Compare commits
243 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 109163621c | |||
| 0ccfbbee5c | |||
| dccab57d39 | |||
| a935e21bf8 | |||
| 3ec6006f57 | |||
| 618d98e822 | |||
| 5897764024 | |||
| 97668d26d3 | |||
| 66a2b0c3e9 | |||
| de8bc7a12c | |||
| 3c3513ea3e | |||
| a5a0496176 | |||
| 2609c5462e | |||
| 68a1f31397 | |||
| aa923cea83 | |||
| e04e1bf127 | |||
| 0d249f3ee9 | |||
| 3d7eea6280 | |||
| 8dfa165110 | |||
| 37390139fe | |||
| 15ccd9b722 | |||
| 942ca7bc14 | |||
| df0c1f3513 | |||
| ba46f4f47a | |||
| 934909b430 | |||
| ebc6b8d4d9 | |||
| f6681544e7 | |||
| 92aab7282c | |||
| 60febbf2dc | |||
| 88141d6ed5 | |||
| 9bb1363cd6 | |||
| 574c3aa1b0 | |||
| d28e3e64f1 | |||
| ccda27d220 | |||
| d277f34a96 | |||
| 186b7d5ea5 | |||
| d33992034d | |||
| 71d031fcd0 | |||
| f0d064a68c | |||
| 830b47f624 | |||
| 7b86059daf | |||
| ab5030174a | |||
| 0e47e887de | |||
| dbffb6bdb4 | |||
| 39552e15b9 | |||
| 1f5d310071 | |||
| a3ee917fb4 | |||
| 8114f13a28 | |||
| 6796ba4126 | |||
| 9e47532283 | |||
| 44a03bedc0 | |||
| 9ced6455ad | |||
| 2d03268975 | |||
| cf1af55720 | |||
| be2f0cc14d | |||
| 410c8d8976 | |||
| 70fb1f1fda | |||
| ce3369ebca | |||
| fe644d7eda | |||
| 4a3b12564a | |||
| 910befc84c | |||
| cc0cc8cd89 | |||
| 549d2d68b3 | |||
| 197246c3c1 | |||
| 9d5eef658f | |||
| 5f225f0b44 | |||
| 78f604e0cd | |||
| dbdf29c8fc | |||
| c0d6bacd9c | |||
| a11ab0dc08 | |||
| 113f1eaf7c | |||
| c019d0fc2b | |||
| ede8610804 | |||
| 6ea3ba44da | |||
| 8ebee2ac40 | |||
| d266ff83f4 | |||
| 2ceed561ad | |||
| 11f792538e | |||
| b5e3e5108c | |||
| b8344bb151 | |||
| dbfe2fc4de | |||
| f6a2628656 | |||
| 0d206a0884 | |||
| b61975f7d7 | |||
| 19a4c093be | |||
| fb5ec80fe6 | |||
| 6d5c1c768b | |||
| beb1c583b1 | |||
| 945ec2c069 | |||
| 7349260231 | |||
| 50c09e21d5 | |||
| 27bf4826d6 | |||
| 029a916288 | |||
| 232c03ed1b | |||
| 7594120a55 | |||
| 93d6c1da67 | |||
| 30a9d5f094 | |||
| fccffd0a0f | |||
| 9120182314 | |||
| 27ca6cd791 | |||
| 8e6ac4b344 | |||
| e59ab69398 | |||
| 8d154c6a81 | |||
| 0658d244d0 | |||
| 69190b14f1 | |||
| 1375f9349e | |||
| 045a21da14 | |||
| f292cf3beb | |||
| 3067146be3 | |||
| fe919f8b08 | |||
| f0b409f999 | |||
| f9ab2f7293 | |||
| 1e343b0c52 | |||
| 42eb96af9e | |||
| f41df6710c | |||
| 825393299e | |||
| bc03adb1fd | |||
| 099bc1b6e0 | |||
| 57deaeba16 | |||
| 7c80d36eb6 | |||
| 062086f9a6 | |||
| cfd403710e | |||
| 699871040a | |||
| 87e8aff786 | |||
| 984e069f5d | |||
| e583b068a1 | |||
| 99c13e4459 | |||
| 3b739b982b | |||
| 56bb4855bb | |||
| 1f88669304 | |||
| db6446ce73 | |||
| b028322568 | |||
| d803e0eab4 | |||
| 50b504d293 | |||
| acce99de9a | |||
| 91434e7f61 | |||
| b3db64af6c | |||
| e4123ada91 | |||
| 2a8f9af5e4 | |||
| 1b2c36eb87 | |||
| 924eff8d3c | |||
| f9fd495292 | |||
| 7d75478f10 | |||
| e5d12d391a | |||
| db6a53036b | |||
| ee67b6370a | |||
| 1569a59ad2 | |||
| 2ebe0a4226 | |||
| 8d82d049e9 | |||
| e177a50046 | |||
| 94691ec50f | |||
| 17cff2ba16 | |||
| f661c084d8 | |||
| 0e2bffc74a | |||
| 3244f9781a | |||
| d2be213e86 | |||
| 13d628a07f | |||
| 18e85265f9 | |||
| 3790a21acb | |||
| 9d90dc9457 | |||
| e5b9783ba3 | |||
| c4b4b2e2bc | |||
| 5c0671e5b5 | |||
| 4c053cc4d0 | |||
| ee174c0179 | |||
| ecc0f914cc | |||
| fabcc6eed6 | |||
| 11b703708c | |||
| c5b33aa123 | |||
| ca2c20f852 | |||
| 082e808d0d | |||
| f0700571e8 | |||
| 8c6d6aff6a | |||
| 27991434a0 | |||
| cdea173bd1 | |||
| f54ab04e64 | |||
| b3031dfea9 | |||
| 9998d816f1 | |||
| 8ec7a83c83 | |||
| 5fdf205caa | |||
| e2011a653e | |||
| 159d18de59 | |||
| 60a1e7e669 | |||
| 7fafeb433c | |||
| f8ad74dc19 | |||
| c6d4134795 | |||
| 96ec751a0b | |||
| 7c787b8b64 | |||
| 6e10e45bb9 | |||
| 8d51380ff1 | |||
| 821763a09d | |||
| fc57eefca6 | |||
| cc34daf181 | |||
| 71fe3a11a7 | |||
| 7500106c2f | |||
| a611cc358f | |||
| 0a8deba94c | |||
| a8c0919365 | |||
| d0876440df | |||
| a4606246b3 | |||
| 8f859acb2c | |||
| c2ab68ceb5 | |||
| d6afb39413 | |||
| 38d37f5c23 | |||
| bef47f89da | |||
| 2d4fbc857f | |||
| 32852382cc | |||
| 31d342eff3 | |||
| 65e6352dad | |||
| 1447a50140 | |||
| 08c340add7 | |||
| 4fa6cf8fee | |||
| 31239d51ff | |||
| cc085cde84 | |||
| 2ef8ee3ade | |||
| 1c822eeeed | |||
| 17962d4233 | |||
| 9973ae08be | |||
| 29893ef1b4 | |||
| 04b882f5c6 | |||
| 70dde115f0 | |||
| 8c8e243de6 | |||
| ff7c6c531a | |||
| a6643cedf9 | |||
| ba727da2dd | |||
| 88ef1ea908 | |||
| 1ea1da8236 | |||
| f622d0be2d | |||
| 97fcf624ae | |||
| 1355049f50 | |||
| 39c5af3d7f | |||
| 83d129d1bd | |||
| 28ca04e466 | |||
| 5067509dd4 | |||
| ea220363f3 | |||
| b394413e30 | |||
| ede40b6c6d | |||
| 24eed8addd | |||
| a7c27e978f | |||
| c36224d956 | |||
| 1f983d610f | |||
| 48a80b5962 | |||
| c0fbe74693 |
@@ -43,8 +43,8 @@ through.
|
||||
([`../HistorianGateway/CLAUDE.md`](../HistorianGateway/CLAUDE.md)) — single-process
|
||||
gRPC sidecar exposing the AVEVA Historian read/write API + read-only Galaxy browse
|
||||
(via the shared `ZB.MOM.WW.GalaxyRepository` lib). Patterned on `mxaccessgw` but with
|
||||
no COM / no x86 worker. **OtOpcUa's historian backend** (`Driver.Historian.Gateway`)
|
||||
consumes it as the sole historian path.
|
||||
no COM / no x86 worker. **OtOpcUa's default historian backend** (`Driver.Historian.Gateway`)
|
||||
consumes it; `ServerHistorian:Backend=SqlServer` selects a plain SQL Server historian instead (see Historian section).
|
||||
- **`~/Desktop/ScadaBridge`**
|
||||
([`../ScadaBridge/CLAUDE.md`](../ScadaBridge/CLAUDE.md)) — the distributed SCADA
|
||||
platform (Akka.NET hub-and-spoke) whose Data Connection Layer consumes this server's
|
||||
@@ -302,7 +302,9 @@ non-`Healthy` state **and** its `LastSuccessfulRead` has been frozen for 90 s. T
|
||||
success. Each conjunct has a test that goes red when the *other* is deleted.
|
||||
- **The mechanism rests on every driver reaching a non-Healthy state on sustained failure.** Surveyed across
|
||||
all 12: nine poll drivers via `Degraded`, MQTT via live connection state, Galaxy via its reconnect
|
||||
supervisor. `CalculationDriver` reports `Healthy` while running and so can never trip — correct, as it has
|
||||
supervisor, OpcUaClient via a bad session keep-alive (#670 — it used to stay `Healthy` forever; its SDK
|
||||
`SessionReconnectHandler` is deleted, a lost session publishes `BadNoCommunication` once per outage and the
|
||||
watchdog's re-init is the recovery; good keep-alives + notifications advance `LastSuccessfulRead`). `CalculationDriver` reports `Healthy` while running and so can never trip — correct, as it has
|
||||
no transport. Re-check this if a new driver lands.
|
||||
- **The reconnect retry MUST tear down before re-initialising — FIXED in both drivers, keep it that way.**
|
||||
`ModbusDriver` (`_transport`) and `MqttDriver` (`_connection`) used to overwrite an `IAsyncDisposable`
|
||||
@@ -319,6 +321,13 @@ non-`Healthy` state **and** its `LastSuccessfulRead` has been frozen for 90 s. T
|
||||
`DefaultReconnectInterval`. Unlike the #523 drift detector this generates **no** traffic on a healthy
|
||||
system — it is a local `GetHealth()` read, and only reaches the network once a driver has already stopped
|
||||
reading.
|
||||
- **Fast path for DEFINITIVE losses (#676):** a driver implementing the opt-in `IDriverConnectionLossSource` raises
|
||||
`ConnectionLost` (once per connection) when its protocol proves the connection can never recover in place — OpcUaClient:
|
||||
keep-alive `BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/`BadServerHalted`, a SUSTAINED closed
|
||||
connection (2nd `BadSecureChannelClosed`/`BadConnectionClosed` in one outage — what a real process stop produces; the
|
||||
SDK never reopens it) or a subscription the SERVER reported dropped (its publishing-stall heuristic does not raise) — and `DriverInstanceActor` (attach PreStart, detach PostStop) turns it into the same disconnect only
|
||||
while Connected; ambiguous failures (timeouts, `BadNoCommunication`, a stalled socket) must never raise and stay with
|
||||
the watchdog's conjunction. `docs/DriverLifecycle.md`.
|
||||
|
||||
- **What comms loss now does downstream — one contract, six seams (archreview R3-02).** "A value consumer
|
||||
never reports a quality better than its source's" is stated once in `docs/DriverLifecycle.md` §"The
|
||||
@@ -1087,7 +1096,7 @@ re-emit a concrete default. The defaults differ per driver (final review, 2026-1
|
||||
because the driver factory AND the admin-node Test Connect probe both read it. ⚠️ So **the upgrade restart
|
||||
alone moves no-key AbCip drivers to Native**; to stay on libplctag, set the node key — NOT a per-driver key
|
||||
through the old AdminUI, whose typed form drops unknown keys (#576). Evidence: AbCip reads proven on a real
|
||||
CompactLogix (Z-35162); scalar native writes (DINT/BOOL/REAL) proven on it once (2026-10-09, user-approved one-off, restored) — and native UDT member writes (DINT/BIT/REAL, #611 gate, same day); array/STRING, libplctag member and fragmented writes are `ab_server`-only.
|
||||
CompactLogix (Z-35162); scalar native writes (DINT/BOOL/REAL) proven on it once (2026-10-09, user-approved one-off, restored) — and native UDT member writes (DINT/BIT/REAL, #611 gate, same day) and a native STRING write (hw5, same day, restored byte-exact); and a native array-element write (`EM01_PowerUnit._SEQ_EP[31]`, user-approved, an element no alias targets, restored; neighbours unchanged); libplctag member and fragmented writes are `ab_server`-only. #566: a real Logix drops TCP on a connection idled past its timeout (no 0x0107/0x0203); the faulted session reopens on the next request.
|
||||
- **AbLegacy: an absent key resolves to `Libplctag` for every family; `Native` is opt-in** (no node key). On
|
||||
a real MicroLogix 1400 (Z-33323, idle, 2026-10-09) native read S/N/F/B/T/C/R/ST/O/I incl. `S:0`, T/C/R
|
||||
sub-elements and status bits, and ST byte order, all correct, and wrote N/F8/B-bit (restored). libplctag (the default)
|
||||
@@ -1151,13 +1160,27 @@ host logger factory (wired in `DriverFactoryBootstrap`; the FOCAS and TwinCAT fa
|
||||
to duplicate them); a flagged `Unspecified` on an already-active condition (AbCip ALMA governing-limit drop-out)
|
||||
moves the node's text/severity, keeps active/acked and writes no row. Active+acked is the atomic **`AlarmTransitionKind.ActiveAcknowledged`** (Galaxy snapshot
|
||||
`ActiveAcked`, OpcUaClient inactive→acked-active) — never Acknowledge (kept a cold host's `Active=false`: the alarm was
|
||||
invisible) nor Raise+Acknowledge (an unacked intermediate). `NativeAlarmProjector` returns the rows to write; unflagged
|
||||
invisible) nor Raise+Acknowledge (an unacked intermediate). Its mirror **`InactiveUnacknowledged`** (#657, OpcUaClient)
|
||||
replaces Raise/Retrigger+Clear (an Active flicker; rows only for what changed). OpcUaClient's first sight of an inactive
|
||||
condition is a flagged InactiveUnacknowledged, or flagged Clear+Acknowledge for inactive+acked (a flagged Acknowledge on an
|
||||
acked condition writes no row) — never `Unspecified`, which wrote "Activated" for an inactive alarm and left a
|
||||
retained-Active condition Active. `NativeAlarmProjector` returns the rows to write; unflagged
|
||||
sources (FOCAS/MTConnect, live Galaxy) are unchanged. E2E: `Runtime.Tests/NativeAlarmRestatementEndToEndTests`.
|
||||
- **Re-Initialize without teardown (#630):** FOCAS, AbLegacy, AbCip, TwinCAT, S7 and OpcUaClient `InitializeAsync`
|
||||
first release the prior device/session state (cancel, ≤ 5 s join, dispose — incl. the AbCip alarm projection;
|
||||
FOCAS's is PAUSED, its told record kept, #647) — the device-state half of `ShutdownAsync` only; the poll engine is kept and the actor re-subscribes
|
||||
after every successful init. AbCip publishes fresh tag tables by reference swap (a second init used to throw
|
||||
"tag name collision" on `AdoptConfigDuringInit`); S7 swaps `Plc` under `_gate`.
|
||||
"tag name collision"); S7 swaps `Plc` under `_gate`.
|
||||
- **Data and alarm subscriptions are independent in `DriverInstanceActor` (#662):** a data re-subscribe (every
|
||||
deploy's bulk push to a kept driver) detaches DATA only (`DetachDataSubscription`) — it used to unhook
|
||||
`OnAlarmEvent` too, silently killing the native-alarm feed until a respawn. The health-poll reconcile
|
||||
re-attaches/re-subscribes alarms as well as data, so a failed alarm subscribe is retried every tick.
|
||||
- **A config change is a respawn — there is NO in-place path (#665).** The `ApplyDelta`/`ApplyResult`/`DeltaApplied`
|
||||
seam (host `ApplyChildDelta`, `DriverSpawnPlan.ToApplyDelta`, the actor's Connected reinit and mid-connect
|
||||
`AdoptConfigDuringInit`) was unreachable since #516 and is deleted; #663/#664 were latent bugs in it. `IDriver.ReinitializeAsync`
|
||||
stays as a per-driver contract the runtime never calls. Every driver's `InitializeAsync` still serves the config it is
|
||||
handed (Calculation adopts, Galaxy refuses a non-equivalent one — #664), and still releases prior state (#630).
|
||||
`docs/DriverLifecycle.md`.
|
||||
- **S7 (#632/#638):** STRING/WSTRING/CHAR/WCHAR and DTL/DATE/TOD/TIME/S5TIME codecs (`S7ValueCodecs`; TOD/TIME/S5TIME
|
||||
surface as Int32 ms); an undecodable buffer or unrepresentable write is `BadOutOfRange` with no health impact.
|
||||
Optional `localTsap`/`remoteTsap`/`connectionType` override the slot TSAP (`remoteTsap` ⊕ `connectionType`). Test
|
||||
@@ -1311,9 +1334,9 @@ are no longer supported (pinned by a test). Condition teardown must forget the w
|
||||
|
||||
## Historian / HistoryRead
|
||||
|
||||
**Backend: HistorianGateway (sole historian backend).** As of the gateway-integration cutover, the
|
||||
historian read, alarm-write, and continuous-historization paths are all served by the
|
||||
**`ZB.MOM.WW.HistorianGateway`** sidecar, consumed as the Gitea-feed
|
||||
**Backend: pluggable — `ServerHistorian:Backend` = `Gateway` (default), `SqlServer` or `QuestDb`** (see the subsection below).
|
||||
As of the gateway-integration cutover, the **Gateway** backend's historian read, alarm-write, and
|
||||
continuous-historization paths are all served by the **`ZB.MOM.WW.HistorianGateway`** sidecar, consumed as the Gitea-feed
|
||||
**`ZB.MOM.WW.HistorianGateway.Client`** gRPC package (`historian_gateway.v1`) behind a thin
|
||||
`IHistorianGatewayClient` seam in `ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway`. **The bespoke
|
||||
Wonderware TCP/ArchestrA sidecar projects and the vestigial `Historian.Wonderware` driver type were
|
||||
@@ -1324,12 +1347,26 @@ A tag is historized by adding `"isHistorized": true` to its `TagConfig` JSON blo
|
||||
raw-JSON textarea on the `/uns` TagModal); an optional `"historianTagname"` field overrides the default
|
||||
historian tagname, which is the tag's driver `FullName`.
|
||||
|
||||
### Pluggable backends, SQL Server and HistoryUpdate (2026-10)
|
||||
|
||||
- **Selection:** `ServerHistorian:Backend` (`Gateway` default | `SqlServer` | `QuestDb`; unknown fails host start). `HistorianBackendRegistry` → `IHistorianBackendProvider` builds all five seams; the gateway keys are only required for `Gateway`. `SqlServer` uses `ServerHistorian:SqlServer:{ConnectionString (required), Schema=hist, CommandTimeout=30s, AutoCreateSchema=true}`.
|
||||
- **SQL Server** (`Driver.Historian.SqlServer`): `hist.Sample` PK `(TagId, SourceTimeUtc)` — writes are insert-if-absent, so outbox redelivery is idempotent. Stores every value kind with its **real status**; per-sample aggregates (10, simple definitions) are pushed down to T-SQL with bucket math in **100 ns ticks**; the Part 13 time-weighted seven (TimeAverage/TimeAverage2/Total/Total2/Interpolative, #671 — ⚠️ `Total` is now the time-weighted integral, no longer `SUM` — plus the **duration-based** PercentGood/PercentBad, #680) stream window + 4 bound rows into the shared `TimeWeightedAggregator` (both SQL backends, identical results; reverse `Interpolative` anchored at the later time; window capped by `ServerHistorian:MaxTimeWeightedRawRows` ⇒ `BadTooManyOperations`, percentages included). ⚠️ **#680 status rules (client-visible):** TimeAverage/Total are Uncertain if any part of the interval uses an Uncertain value, is bridged across a Bad sample, is extrapolated past the last sample (⇒ a live tag's latest interval is Uncertain) or is not covered; TimeAverage2/Total2 keep the SDK time-based rule (Uncertain time counts as Good) but never extrapolate past the last sample — that interval is Partial, later ones `BadNoData` (a report-by-exception tag with nothing after its last sample reads `BadNoData` until its next sample arrives). `docs/Historian.md` §"Time-weighted aggregates". ⚠️ **#680 per-sample (client-visible):** `Count` counts **Good** samples only (value-based status; an all-Bad interval is generic Bad, value null) and an empty interval **inside the tag's data range** is `0`/Good (outside it still `BadNoData`); `Start`/`End` return the **raw** first/last sample of any quality — its own timestamp and status, value null when Bad; `Delta` is Uncertain only when the interval's first or last sample is unusable; ⚠️ #684: `Average`/`Minimum`/`Maximum`/`Range`/`*ActualTime` use **Good samples only** (was Good + Uncertain; an Uncertain-only interval is `BadNoData`), Min/Max-family status Uncertain on a Bad sample, an Uncertain value beyond the Good extreme (Part 13 — the SDK ignores that) or an Uncertain non-numeric sample, and only a **Good** String/DateTime sample refuses them (`BadAggregateInvalidInputs`; Delta still refuses any usable one); and every per-sample aggregate but `Average` sets **end-of-data Partial** on the interval holding the last sample (forward incl. its earlier edge, reverse strictly inside), never on a Bad result. The data-range lookup is a key seek on SQL Server and a `tag` symbol-index seek on QuestDB (#683 — `{p}sample.tag` is `SYMBOL INDEX`; the bootstrapper ADDs it to a pre-#683 table, idempotent), read BEFORE the bucket statement so a racing append can never yield a Good `Count` of 0 (a concurrent HistoryUpdate delete is the mirror case and still can, transiently). `docs/Historian.md` §"Aggregates". Docker-dev recipe, schema and aggregate semantics: `docs/Historian.md` §"SQL Server backend". Integration tests need `HISTORIAN_SQL_TEST_CONNECTION`. **Retention (#667):** opt-in `SqlServer:RetentionDays` (+ `PurgeInterval` 1h, `PurgeBatchSize` 4000) -> `HistorianRetentionService` purges old `hist.Sample` (per tag) / `hist.Event` in `DELETE TOP(n)` batches, Primary-gated on `IHistorizationRoleView`, one purger per DB via `sp_getapplock`; counter `otopcua.historian.retention.purged{table}`. Optional `IHistorianBackendProvider.CreateRetention` (default null). `docs/Historian.md` §Retention.
|
||||
- **QuestDB** (`Driver.Historian.QuestDb`, `ServerHistorian:QuestDb:*`, Npgsql over PG wire; docker-dev overlay `docker-compose.questdb.yml`): same kinds/status/aggregates/HistoryUpdate as SqlServer, soft delete by tombstone (INSERT only — never `UPDATE`/`DELETE`; every read filters `NOT deleted`). ⚠️ **Timestamps are µs** (`QuestDbConnection.Round`/`BindTimestamp`; `IHistorianDataSource.TimestampResolutionTicks` = 10 steps HistoryRead paging), **pre-1970 cannot be stored** (writer drops, HistoryUpdate `BadOutOfRange`), every write waits for WAL visibility, TTL is enforced on write commits only, set Npgsql `Maximum Pool Size` < QuestDB's 64. `docs/Historian.md` §"QuestDB backend".
|
||||
- ⚠️ **Reverse processed reads are Part 13/SDK-anchored (#677, client-visible):** every aggregate on SqlServer/QuestDB now anchors a reverse read at its later time — window `(end, start]`, partial interval earliest, each stamped with its later edge (was: earlier-anchored, stamped earlier edge); on forward reads Partial also marks the interval the data starts inside (not Average — the SDK's rule, forward-only); a non-default client `AggregateConfiguration` ⇒ `BadAggregateConfigurationRejected`; Gateway provisioning sends `IntegralDivisor = 1`. SDK-parity tests: `SdkAggregateParityTests`. `docs/Historian.md` §Aggregates.
|
||||
- ⚠️ **Reverse / end-only RAW reads work on Gateway (#686, client-visible):** `IHistorianDataSource.SupportsReverseRaw` (SqlServer/QuestDB/Null `true`, Gateway `false`, interface default `false`, guard-tested) — the node manager sends `start > end` only to a backend that declares it; otherwise `ReverseRawReader` serves it from FORWARD reads: uncapped ⇒ ONE forward read reversed in memory; capped ⇒ a bounded walk back in segments (ms-aligned request bounds, width aimed from observed density at ≈ N + 8 rows and carried page to page in the continuation point — #689: it used to aim at half `MaxTieClusterOverfetch`, ~32k rows per page on a dense tag; a read asks for `2N + 18` rows (a discovery segment ≤ 1,026), `MaxTieClusterOverfetch` is only the tie-cluster ceiling — a full read is re-read, never answered truncated; only more than max(1, `MaxTieClusterOverfetch`) samples within ONE MILLISECOND ⇒ `BadTooManyOperations` (per ms, not per timestamp — requests are ms-rounded, and letting a segment narrow below 1 ms stepped a page through one ms a tick per read, ~10k reads); walks no later than now + 1 day and never before the Unix epoch; #691: after two empty segments it asks once for the whole remainder, and the stretch a page proved empty rides the continuation point — the last pages of a deep page-through no longer each walk ~14 reads to 1970). The raw/events tie-path oversized-cluster refusal is now `BadTooManyOperations` too (was `BadHistoryOperationUnsupported`). Uncapped Gateway raw reads (forward too) used to send `max_values = 0`, which the client/gateway reject — now unbounded (`int.MaxValue`; a ceiling refusal ⇒ `BadTooManyOperations`), and a cancelled read is no longer a backend failure. Since #669 such reads failed the node on Gateway and counted as a backend fault (the gateway rejects start > end — `GatewayHistorianDataSource.ReadRawAsync` now refuses one unsent); before #669 an end-only Gateway read returned the OLDEST N ascending — now the latest N at/before EndTime, newest first. Events follow since #691.4: a reverse/end-only capped events read on Gateway (`SupportsReverseEvents` false; Null now reports `true` and answers empty) is walked back the same way (ties sorted by EventId — the gateway orders by time only; one extra event found ⇒ continuation point) — it used to be refused `BadHistoryOperationUnsupported` (explicit reverse) or answered with the OLDEST N ascending (end-only); needs the gateway's `RuntimeDb:EventReadMaxRows` 0/unlimited (it truncates silently). Uncapped Gateway events reads (forward too) used to throw "Array dimensions exceeded" (result list pre-sized to `int.MaxValue`) — fixed. Above a truncated read the walk gallops up from the newest row seen, then bisects (never below 1 ms): a burst that ended 100 days ago ≈ 18 reads, a dense tag that stopped days ago ≈ 26. `docs/Historian.md` §"Open-ended and reverse raw reads".
|
||||
- **Advertised aggregates (#682):** `IHistorianDataSource.SupportedAggregates` (Gateway 4 via `GatewayHistorianDataSource.ServedAggregates`, SqlServer/QuestDB 17 via `AggregateFamilies.Served`, Null **none**; a null set ⇒ all) is the one authority for `HistoryServerCapabilities/AggregateFunctions` (i=11201 — only its folder references are trimmed) and the up-front `BadAggregateNotSupported` in HistoryRead-Processed. ⚠️ `ServerCapabilities/AggregateFunctions` (i=2997) is deliberately NOT trimmed: it keeps the SDK's 37 because live `AggregateFilter` subscriptions still compute them all. ⚠️ Historian disabled ⇒ Processed is now `BadAggregateNotSupported`, not `GoodNoData`. `docs/Historian.md` §Advertised aggregates.
|
||||
- **Raw `ReturnBounds` is honoured on every backend (#691, client-visible):** Part 11 bounding values (§4.6 Table 1 is the test oracle) are composed in the server by `HistoryBounds` from each backend's raw reads — "at or before" = capped reverse read (native on SQL backends, the #686 `ReverseRawReader` walk-back on Gateway, never AVEVA's synthetic boundary row), "at or after" = capped forward read. Bounds are stored samples of any quality, count toward `NumValuesPerNode`, start bound on the first page / end bound on the last (pending end bound rides the continuation point); missing ⇒ `Bad_BoundNotFound` (null, at the bound time; a one-sided read's open side ±1 s from the previous value). ⚠️ **Breaking for SDK-default clients:** the .NET SDK's `new ReadRawModifiedDetails()` defaults BOTH `ReturnBounds = true` AND `IsReadModified = true` — a client that cleared only `IsReadModified` now gets bounds, and one that set neither is a modified read with bounds ⇒ `Bad_InvalidArgument` (§6.5.3.3); set both explicitly (Client.Shared does; tests building details must too). No data AND no bound found ⇒ `GoodNoData`, no values (the pre-bounds answer). On Gateway the at-or-after lookup ends at now + 1 day (never a year-9999 end to AVEVA). Kill switch `ServerHistorian:HonorReturnBounds` (default `true`; `false` = pre-#691). `docs/Historian.md` §Bounding values.
|
||||
- **Real status fix:** the recorder used to stamp every sample Good (`192`); it now carries the driver's status. ⚠️ On the **Gateway** backend this changes nothing stored: the gateway discards quality (C-002), so AVEVA still stores Good.
|
||||
- **Comms loss is historized as an outage marker** (`Empty` + `BadNoCommunication`, one per tag per outage) on SqlServer; dropped (`DroppedUnsupportedKind`) on Gateway. Outbox records are **format v2** — drain or delete `ContinuousHistorization:OutboxPath` before downgrading a node.
|
||||
- **HistoryUpdate** (SqlServer + QuestDb; Gateway ⇒ `BadHistoryOperationUnsupported`): Insert/Replace/Update/DeleteRaw/DeleteAtTime, **Primary-only** since #674 (off-Primary ⇒ `BadInvalidState`, after authorization AND after planning — an unservable request (Remove / `IsDeleteModified` / not historized) stays `BadHistoryOperationUnsupported` off-Primary; the verdict is `IHistoryUpdateRoleView` = the data-plane verdict but **closed until this node's first redundancy snapshot**, so a pair cannot both accept in the boot window; admin-only nodes register no view ⇒ no gate; reads ungated), items with a `SourceTimestamp` past now + `ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day, ≤0 disables) ⇒ `BadOutOfRange` in the node manager (every backend), gated in both ACL modes — flag off needs the `HistoryUpdate` role from a **Cluster-scoped** grant (+ key re-check), flag on uses `DataPlaneAuthorizer`'s HistoryUpdate operation; `WriteOperate` never implies it. The `HistoryWrite` AccessLevel bit is set only when an updater is wired at materialise time. `docs/Historian.md` §HistoryUpdate, `docs/security.md`. ⚠️ **SqlServer schema v1 is accepted (#674)** with `AutoCreateSchema=false`: writes/reads/events/retention work; modified reads + HistoryUpdate ⇒ node-level `BadHistoryOperationUnsupported` on every arm (the updater throws `NotSupportedException`, nothing sent; the `HistoryWrite` bit is still advertised — not knowable at materialise time), retention skips `SampleModified`, one Warning per boot (`SqlServerSchemaBootstrapper.SupportsModifiedTrail`; `RequiredVersion` = 1, `ModifiedTrailVersion` = 2). QuestDB: keep `cairo.ttl.use.wall.clock=true`; a QuestDB trail INSERT retried after a client timeout duplicates (no DEDUP key).
|
||||
- **Modified-value history (#668, Part 11):** every applied HistoryUpdate item leaves a trail row (prior value, principal from the session's API key / `anonymous`, `HistoryModificationType`, time) — SqlServer `hist.SampleModified` (**schema v2**, written by `OUTPUT … INTO` in the same statement ⇒ atomic; ⚠️ no FK/CHECK/trigger on it, and `AutoCreateSchema=false` installs must run `002_sample_modified.sql` before upgrading), QuestDB `{p}sample_modified` (append-only, written **before** the change ⇒ a crash leaves an orphan trail row, never an untracked change). `IsReadModified=true` HistoryRead serves it as `HistoryModifiedData` via `IHistorianDataSource.ReadModifiedAsync` (default ⇒ `NotSupported` ⇒ `BadHistoryOperationUnsupported` on Gateway/Null), paged on `(SourceTime, ModifiedId)`, gated by plain HistoryRead. Live writes and retention never write it; retention/TTL purge it on source time. `IHistorianDataUpdater` methods take a `HistoryUpdateContext`. CLI `historyread --modified`. `docs/Historian.md` §Modified-value history.
|
||||
|
||||
### Read path (`ServerHistorian` section)
|
||||
|
||||
The server dispatches all OPC UA HistoryRead to the registered `IHistorianDataSource` — the
|
||||
`GatewayHistorianDataSource` read client when enabled, else the `NullHistorianDataSource` default
|
||||
(historized nodes return `GoodNoData`, never an error). Supported variants: Raw, Processed
|
||||
(Average/Minimum/Maximum/Total/Count aggregates), and AtTime over historized variable nodes; Events over
|
||||
(historized nodes return `GoodNoData` for Raw/AtTime — Processed is `BadAggregateNotSupported` since #682). Supported variants: Raw (⚠️ #687, client-visible: Gateway raw reads no longer return AVEVA's synthetic start-boundary row — asked from 1 ms early (epoch-floored) with `NumValuesPerNode + 16` rows (unclamped; 0 ⇒ `int.MaxValue`) and filtered by timestamp — so an empty window is `GoodNoData`, not a fabricated value at StartTime; the forward end is now exclusive (a point read inclusive); pre-window rows that crowd the budget trigger ONE re-read with room for them, else `BadTooManyOperations`; `ReturnBounds` is ignored on every backend; ⚠️ #690: a Gateway Bad row — a stored NULL, which histsdk sends as value 0 — now reads value **null** + Bad on Raw and AtTime, as on the SQL backends; ⚠️ #691.3: a strict gateway (`AllowStreamingBeyondCeiling=false`) that refuses a raw budget is re-asked ONCE at the ceiling its refusal names — raw and per-sample reads alike; the ceiling is then remembered per data source (`GatewayRowCeiling`: refusals count as gateway errors), re-learned from a refusal when lowered, and re-probed once at the full budget when a ceiling-bounded read comes back full (raised / streaming) — so an uncapped (`NumValuesPerNode = 0`) read of a small window is served; a ceiling-bounded stream that is not provably complete is `BadTooManyOperations`, never a truncated answer), Processed
|
||||
(backend-dependent: SqlServer/QuestDb serve seventeen; Gateway twelve = `GatewayHistorianDataSource.ServedAggregates` — ⚠️ #681, client-visible: the ten per-sample ones (Average, Count, Min/Max, Range, Start/End/Delta, Min/MaxActualTime) are evaluated from RAW gateway reads by the shared `Sql.Common` evaluator — identical to the SQL backends, Calculated bits, interval-start stamps, #677 reverse; Min/Max are no longer AVEVA modes, the read moves every raw sample of the window (a month of 1 Hz ⇒ `BadTooManyOperations` at the default `MaxTimeWeightedRawRows` cap), and every gateway stream is consumed to completion (an abandoned one makes the gateway evict its shared historian session) — plus TimeAverage/Total via AVEVA modes (interval-start stamped + Calculated; ⚠️ #685: AVEVA's extra pre-window row is dropped ⇒ one row per interval in `[Start, End)`, and an Uncertain (opc 64) interval keeps its value as `UncertainDataSubNormal` — AVEVA's TWA excludes NULL-gap time where the SQL backends bridge it; per-interval values still need gateway fix historiangw#27; ⚠️ #691.2: native reads use the SQL backends' grid in both directions via forward requests only — a REVERSE read (refused by the gateway before) is anchored at its later time (#677), and the clipped interval (earliest reverse, LATEST forward — which used to be averaged past EndTime) is its own exact-span read marked Partial unless Bad; an interval wider than the window is sent as the window, interval 0 as one interval; more than 100,000 intervals is refused up front (`BadHistoryOperationUnsupported`, as on the SQL backends — forward used to be served up to the gateway's 1,000,000); until historiangw#27 lands both directions return Good-status wrong values); the other time-weighted five stay `BadAggregateNotSupported` because the gateway has no prior-sample read for Part 13 bounds; `docs/Historian.md` §Aggregate set), and AtTime over historized variable nodes; Events over
|
||||
alarm-owning equipment-folder event-notifier nodes. Reads are ungated (served from any redundancy node);
|
||||
authorization uses the standard `AccessLevels.HistoryRead` bit set at materialization.
|
||||
|
||||
@@ -1345,8 +1382,10 @@ authorization uses the standard `AccessLevels.HistoryRead` bit set at materializ
|
||||
| `CaCertificatePath` | `null` | PEM CA file pinning the gateway TLS chain; null/empty uses the OS trust store |
|
||||
| `CallTimeout` | `00:00:30` | Per-call deadline for each unary gateway read |
|
||||
| `MaxTieClusterOverfetch` | `65536` | Bounded over-fetch the HistoryRead-Raw paging uses to page within an oversized same-timestamp tie cluster (retained from the prior backend) |
|
||||
| `HonorReturnBounds` | `true` | #691 kill switch: `false` ignores HistoryRead-Raw `ReturnBounds` as before (no bounding values, no modified-read `Bad_InvalidArgument`) |
|
||||
| `HistoryReadBatchParallelism` | `4` | Max nodes served concurrently within one HistoryRead batch (03/S3); `⌈N/P⌉ × CallTimeout` worst case instead of sequential |
|
||||
| `MaxConcurrentHistoryReadBatches` | `16` | Process-wide concurrent HistoryRead batch cap (03/S3); flood degrades to `BadTooManyOperations`. Total in-flight reads capped at 16 × 4 = 64 |
|
||||
| `MaxTimeWeightedRawRows` | `1000000` | `SqlServer`/`QuestDb` (#671): raw in-window rows one time-weighted aggregate read (incl. `Total` and, since #680, PercentGood/PercentBad) may stream into the C# evaluator — and, since #681, every `Gateway` per-sample read (raw-evaluated); window query capped at this + 1, past it ⇒ `BadTooManyOperations` for that node. ≤ 0 = unbounded. Gateway (#688): the `cap + 18` budget is clamped at the gateway's default `MaxValuesCeiling` (1,000,000) while the cap is ≤ it, so the default no longer trips a strict (`AllowStreamingBeyondCeiling=false`) gateway |
|
||||
| `HistoryReadDeadline` | `00:01:00` | Per-request HistoryRead wall-clock deadline (03/S3); a node still in flight surfaces `BadTimeout`. Non-positive = unbounded |
|
||||
|
||||
### Alarm-history path (`AlarmHistorian` section)
|
||||
@@ -1385,6 +1424,7 @@ only the recorder + outbox knobs:
|
||||
| `MaxWriteAttempts` | `5` | #538: consecutive failed drain passes one tag may accumulate — counted **only across passes where other tags wrote successfully** — before its queued entries are abandoned so the FIFO head can advance |
|
||||
| `MinBackoffSeconds` | `1` | Initial retry backoff after a failed drain pass |
|
||||
| `MaxBackoffSeconds` | `30` | Cap on the exponential retry backoff |
|
||||
| `CaptureWhilePrimaryUnreachable` | `true` | #672: a standby stood down for its mesh Primary (#537) captures as soon as that Primary is reported UNREACHABLE, instead of after auto-down — crash-failover gap ~30 s ⇒ ~10–12 s, at the cost of duplicate windows (see #537/#672 below). `false` = pre-#672 |
|
||||
|
||||
### Tag auto-provisioning (`IHistorianProvisioning` EnsureTags hook)
|
||||
|
||||
@@ -1392,9 +1432,10 @@ only the recorder + outbox knobs:
|
||||
hook for added historized value tags — and, since #600.1, for **changed** raw tags that flip to historized or change their resolved historian name or DataType — the gateway-backed `GatewayTagProvisioner` calls the gateway's
|
||||
`EnsureTags` so a brand-new historized tag exists in the historian before the recorder's `WriteLiveValues`
|
||||
land. The hook is wrapped so a faulted/synchronously-throwing provisioner can **never** block or fail a
|
||||
deploy. Non-numeric (`String`/`DateTime`/`Reference`) data types are skipped (not provisioned); the
|
||||
recorder likewise drops + meters non-numeric values. Continuous historization is **numeric-analog only** in
|
||||
v1 (`UInt16→UInt4` is a documented fallback).
|
||||
deploy. **Gateway backend:** non-numeric (`String`/`DateTime`/`Reference`) data types are skipped (not
|
||||
provisioned), the recorder likewise drops + meters non-numeric values, and continuous historization is
|
||||
**numeric-analog only** (`UInt16→UInt4` is a documented fallback). The **SqlServer** backend provisions and
|
||||
records every type (see "Pluggable backends" above).
|
||||
|
||||
The provisioner is registered by the Host's `AddHistorianProvisioning` (`Runtime.ServiceCollectionExtensions`),
|
||||
built via `GatewayHistorian.CreateProvisioner`, and **gated on the same `ServerHistorian:Enabled` flag as the
|
||||
@@ -1416,6 +1457,9 @@ The target HistorianGateway OtOpcUa points at **must** run with:
|
||||
|
||||
- `RuntimeDb:Enabled=true` — enables the `WriteLiveValues` SQL live path (continuous historization).
|
||||
- `RuntimeDb:EventReadsEnabled=true` — enables `ReadEvents` from `Runtime.dbo.Events` (alarm-history reads).
|
||||
- `RuntimeDb:EventReadMaxRows=0` (its default, unlimited) — or larger than the reverse-events walk's biggest read
|
||||
(`2·MaxTieClusterOverfetch + 4`). The cap truncates **silently**, so a walked-back reverse events read (#691.4) would
|
||||
lose the newest events of a segment it cut.
|
||||
- An API key carrying scopes **`historian:read`**, **`historian:write`**, **`historian:tags:write`**.
|
||||
|
||||
### Migration note (deployments upgrading from the Wonderware backend)
|
||||
@@ -1523,6 +1567,21 @@ tag); evidence in the same plan doc. Two things to know before touching either.*
|
||||
`RedundancyRoleView` behind **`IHistorizationRoleView`**, registered unconditionally on driver nodes
|
||||
(NOT alongside the alarm `IRedundancyRoleView`, which only exists when that section is enabled — the
|
||||
#511 registration accident).
|
||||
- **#672 — an UNREACHABLE recording Primary does not count (crash-failover gap).** A SIGKILLed Primary
|
||||
stays a member (unreachable, not downed) for 10 s failure detection + 15 s auto-down, and the
|
||||
redundancy singleton died with it, so the standby's cached snapshot kept naming it Primary and
|
||||
*nobody* captured for ~30 s. `DriverHostActor` now holds its **own** `ReachabilityEvent` (+`MemberUp`)
|
||||
subscription — not inside `StartPrimaryVerdictPublishing`, which early-returns with no primary-role
|
||||
source — handled in **all three** behaviours, caches the last snapshot, and on a reachability change
|
||||
re-derives and republishes **only** the historization verdict with unreachable peers excluded. Gap ⇒ the
|
||||
failure-detector window (~10–12 s). Gated by `ContinuousHistorization:CaptureWhilePrimaryUnreachable`
|
||||
(default `true`). ⚠️ **The alarm drain and `ShouldServiceAsPrimary`/`PublishPrimaryVerdict` must NOT
|
||||
adopt this** (a false allow there is a dual claimant, not a duplicate) — the tests assert both stay
|
||||
put. ⚠️ A **removed** member deliberately stays marked unreachable (only `MemberUp` at that address
|
||||
clears it): un-marking it would re-derive from a cached snapshot still naming it Primary and close the
|
||||
gate on a dead node. Cost: duplicates while a live Primary is merely unreachable and under a real
|
||||
partition — idempotent on SqlServer/QuestDB only for source-timestamped drivers (Modbus stamps each
|
||||
node's own poll time ⇒ duplicate rows on every backend); Gateway stores duplicates.
|
||||
|
||||
Re-confirmed live in 2026-06 and **since fixed (#600.1)**: `EnsureTags` provisioning used to fire only for
|
||||
newly-ADDED historized tags, so flipping an existing tag's `historianTagname` (or historization, or DataType)
|
||||
|
||||
@@ -27,6 +27,9 @@
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Contracts/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Contracts.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ZB.MOM.WW.OtOpcUa.Driver.Modbus.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing.csproj" />
|
||||
<Project Path="src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts.csproj" />
|
||||
@@ -100,6 +103,11 @@
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser.Tests/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browser.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing.Tests/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Addressing.Tests.csproj" />
|
||||
<Project Path="tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests.csproj" />
|
||||
|
||||
+30
-1
@@ -160,6 +160,33 @@ and the script fails loudly saying so. A plain `docker compose up -d` also hides
|
||||
rig without the gate helping: `up -d` blocks on the `depends_on: service_healthy` conditions and
|
||||
returns ~30 s *after* Traefik starts, by which time the router table is long loaded.
|
||||
|
||||
## QuestDB historian overlay
|
||||
|
||||
`docker-compose.questdb.yml` is an **optional overlay** (not part of the default rig) that adds one service, `questdb`
|
||||
(`questdb/questdb:10.0.1`, container `otopcua-dev-questdb-1`, named volume `otopcua-questdb-data`), for the `ServerHistorian:Backend=QuestDb`
|
||||
historian ([`docs/Historian.md`](../docs/Historian.md#questdb-backend)). Host ports: PG wire **`18812`** (→ 8812) and the web console / REST **`19000`**
|
||||
(→ 9000), chosen so they do not collide with AdminUI `:9000`/`:9200`.
|
||||
|
||||
```bash
|
||||
# Bring up ONLY questdb (--no-deps: nothing else is touched):
|
||||
docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d --no-deps questdb
|
||||
|
||||
# Point the central nodes at it (the QuestDb connection string already defaults to the `questdb` service;
|
||||
# override with OTOPCUA_HISTORIAN_QUESTDB). Keep the overlay -f flag on every compose command for these nodes:
|
||||
OTOPCUA_HISTORIAN_ENABLED=true OTOPCUA_HISTORIAN_BACKEND=QuestDb OTOPCUA_CONTHIST_ENABLED=true \
|
||||
docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d central-1 central-2
|
||||
|
||||
# Inspect from the host (REST, no client needed):
|
||||
curl -G localhost:19000/exec --data-urlencode "query=SELECT tag, count() n FROM hist_sample WHERE NOT deleted GROUP BY tag"
|
||||
|
||||
# Integration tests (skipped when the variable is absent):
|
||||
export HISTORIAN_QUESTDB_TEST_CONNECTION='Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading'
|
||||
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb.IntegrationTests
|
||||
```
|
||||
|
||||
Tear down with `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml rm -sf questdb` (add `docker volume rm otopcua-questdb-data`
|
||||
to drop the data). The `admin`/`quest` credentials are QuestDB's stock defaults, not secrets.
|
||||
|
||||
## Tear down
|
||||
|
||||
```bash
|
||||
@@ -180,7 +207,9 @@ The `-v` drops the SQL volume; remove it to keep ConfigDb state across restarts.
|
||||
|
||||
The full single-mesh stack (`central-1`/`central-2` + the four site nodes) can OOM-kill `central-1` on a loaded host. Two settings in the compose file guard against that:
|
||||
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and in `Development` the default level is `Debug` — without these overrides every Deployment-poll emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
- **EF Core + ASP.NET Core logs are pinned to `Warning`** on every host node (`Serilog__MinimumLevel__Override__Microsoft.EntityFrameworkCore` / `…Microsoft.AspNetCore` = `Warning`). The host logs via Serilog (`AddZbSerilog` → `ReadFrom.Configuration`), and a `Development` environment would default to `Debug` (`appsettings.Development.json`) — the rig does NOT set `ASPNETCORE_ENVIRONMENT`, so nodes actually run at the base `Information` default, but these pins keep the SQL chatter out should the rig ever run as Development; without them a Debug/Information-level EF pipeline emits an `Executed DbCommand` / `SELECT … FROM [Deployment]` line, flooding the Serilog pipeline and starving the Akka cluster heartbeat thread. Application + Akka log levels are left untouched, so this only silences the per-poll SQL chatter. To temporarily restore the SQL log flood for debugging, drop those two env vars (or set them back to `Information`) on the node you're inspecting.
|
||||
|
||||
To raise Debug logging for chosen categories on `central-1`/`central-2` (e.g. `ZB.MOM.WW.OtOpcUa.Runtime`), use the overlay: `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2` — it sets `Serilog__MinimumLevel__Override__<Namespace>=Debug`; edit its category list to taste.
|
||||
- **Each host node has `mem_limit: 1g`** (`mem_reservation: 512m`). A quiet solo `central-1` measures ~357 MiB; the limit leaves headroom for the deploy/UI load and per-cluster driver subscriptions that push a fully-loaded node higher. The limit/reservation live on the `&otopcua-host` anchor, so all six host services inherit them; `sql`, `traefik`, and the one-shot `migrator`/`cluster-seed` are left unbounded.
|
||||
|
||||
The full six-node host stack therefore needs roughly **6 GiB** of Docker Desktop VM memory just for the host nodes (plus SQL Server's own footprint on top). On a constrained host, either raise the Docker Desktop VM memory or run fewer host services (e.g. just `central-1` + `central-2`, or a single central node) rather than the full mesh.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# docker-dev/docker-compose.debug.yml — optional overlay: raise Serilog levels per category on the central pair.
|
||||
#
|
||||
# The rig nodes run with ASPNETCORE_ENVIRONMENT unset (Production), so there is NO appsettings.Development.json and the
|
||||
# Serilog default level is Information (the base appsettings.json sets none). Debug is therefore OFF by default; this
|
||||
# overlay turns it on for chosen namespaces only, via Serilog:MinimumLevel:Override:<Namespace> (env: double underscore
|
||||
# for ':', and the namespace's own dots stay dots).
|
||||
#
|
||||
# Up (recreates only the two central nodes with the extra env):
|
||||
# docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.debug.yml up -d --no-deps central-1 central-2
|
||||
# Back to normal: re-run `up -d --no-deps central-1 central-2` with the base file alone.
|
||||
#
|
||||
# Edit the list below to the categories you care about. Useful ones:
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime actors: DriverHostActor, OpcUaPublishActor, redundancy, deploys
|
||||
# ZB.MOM.WW.OtOpcUa.Runtime.Drivers.DriverHostActor one actor only (e.g. the #672 historization verdict logs)
|
||||
# ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient a single driver
|
||||
# Opc.Ua OPC Foundation stack (very chatty)
|
||||
# Akka cluster/remoting internals (chatty; base is Information)
|
||||
# Leave Microsoft.EntityFrameworkCore / Microsoft.AspNetCore at Warning (see README "Resource limits & dev logging"):
|
||||
# a Debug SQL flood starves the Akka heartbeat thread.
|
||||
services:
|
||||
central-1:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
central-2:
|
||||
environment:
|
||||
Serilog__MinimumLevel__Override__ZB.MOM.WW.OtOpcUa.Runtime: "Debug"
|
||||
@@ -0,0 +1,28 @@
|
||||
# docker-dev/docker-compose.questdb.yml — optional QuestDB fixture for the QuestDB historian backend (#666).
|
||||
#
|
||||
# Up (touches ONLY the questdb service; it has no depends_on, and --no-deps makes that explicit):
|
||||
# docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d --no-deps questdb
|
||||
#
|
||||
# Image pinned by the T0 spike (docs/plans/2026-10-10-questdb-spike.md): questdb/questdb:10.0.1,
|
||||
# pushed to Docker Hub 2026-08-24 (≥ 2 weeks before 2026-10-10; ≥ 8.2.3, the first release with table TTL).
|
||||
# The image ships /usr/bin/curl, so the healthcheck uses it against the min-HTTP health endpoint on :9003.
|
||||
#
|
||||
# PG wire -> localhost:18812 (Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;...)
|
||||
# Console -> http://localhost:19000
|
||||
services:
|
||||
questdb:
|
||||
image: questdb/questdb:10.0.1
|
||||
container_name: otopcua-dev-questdb-1
|
||||
ports:
|
||||
- "18812:8812" # PG wire
|
||||
- "19000:9000" # web console / REST
|
||||
volumes:
|
||||
- otopcua-questdb-data:/var/lib/questdb
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:9003/status || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
volumes:
|
||||
otopcua-questdb-data:
|
||||
name: otopcua-questdb-data
|
||||
@@ -434,6 +434,18 @@ services:
|
||||
# locally-run gateway is usually plaintext h2c while a deployed one is https.
|
||||
ServerHistorian__UseTls: "${OTOPCUA_HISTORIAN_USETLS:-true}"
|
||||
ServerHistorian__AllowUntrustedServerCertificate: "true"
|
||||
# Backend selection (pluggable historian, 2026-10). Gateway = the HistorianGateway keys above; SqlServer =
|
||||
# a plain SQL Server historian. For the rig, the `sql` container hosts it (db OtOpcUaHistorian, auto-created
|
||||
# schema `hist` — create the database first):
|
||||
# OTOPCUA_HISTORIAN_ENABLED=true OTOPCUA_HISTORIAN_BACKEND=SqlServer OTOPCUA_CONTHIST_ENABLED=true \
|
||||
# docker compose -f docker-dev/docker-compose.yml up -d central-1 central-2
|
||||
ServerHistorian__Backend: "${OTOPCUA_HISTORIAN_BACKEND:-Gateway}"
|
||||
ServerHistorian__SqlServer__ConnectionString: "${OTOPCUA_HISTORIAN_SQL:-Server=sql,1433;Database=OtOpcUaHistorian;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;Encrypt=False}"
|
||||
# QuestDb = the QuestDB historian; the `questdb` service comes from the docker-dev/docker-compose.questdb.yml overlay
|
||||
# (add `-f docker-dev/docker-compose.questdb.yml` and OTOPCUA_HISTORIAN_BACKEND=QuestDb).
|
||||
ServerHistorian__QuestDb__ConnectionString: "${OTOPCUA_HISTORIAN_QUESTDB:-Host=questdb;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading}"
|
||||
# Empty = no TTL (data kept forever); a positive day count sets the sample/event table TTL.
|
||||
ServerHistorian__QuestDb__RetentionDays: "${OTOPCUA_HISTORIAN_QUESTDB_RETENTION_DAYS:-}"
|
||||
# Continuous historization of driver tag values (Gitea #491's live gate). Gated on BOTH
|
||||
# ContinuousHistorization:Enabled AND ServerHistorian:Enabled — with either off no recorder is
|
||||
# spawned, so this stays inert on the default rig. OutboxPath is the FasterLog crash boundary and
|
||||
@@ -560,6 +572,18 @@ services:
|
||||
ServerHistorian__ApiKey: "${ServerHistorian__ApiKey:-}"
|
||||
ServerHistorian__UseTls: "${OTOPCUA_HISTORIAN_USETLS:-true}"
|
||||
ServerHistorian__AllowUntrustedServerCertificate: "true"
|
||||
# Backend selection (pluggable historian, 2026-10). Gateway = the HistorianGateway keys above; SqlServer =
|
||||
# a plain SQL Server historian. For the rig, the `sql` container hosts it (db OtOpcUaHistorian, auto-created
|
||||
# schema `hist` — create the database first):
|
||||
# OTOPCUA_HISTORIAN_ENABLED=true OTOPCUA_HISTORIAN_BACKEND=SqlServer OTOPCUA_CONTHIST_ENABLED=true \
|
||||
# docker compose -f docker-dev/docker-compose.yml up -d central-1 central-2
|
||||
ServerHistorian__Backend: "${OTOPCUA_HISTORIAN_BACKEND:-Gateway}"
|
||||
ServerHistorian__SqlServer__ConnectionString: "${OTOPCUA_HISTORIAN_SQL:-Server=sql,1433;Database=OtOpcUaHistorian;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;Encrypt=False}"
|
||||
# QuestDb = the QuestDB historian; the `questdb` service comes from the docker-dev/docker-compose.questdb.yml overlay
|
||||
# (add `-f docker-dev/docker-compose.questdb.yml` and OTOPCUA_HISTORIAN_BACKEND=QuestDb).
|
||||
ServerHistorian__QuestDb__ConnectionString: "${OTOPCUA_HISTORIAN_QUESTDB:-Host=questdb;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading}"
|
||||
# Empty = no TTL (data kept forever); a positive day count sets the sample/event table TTL.
|
||||
ServerHistorian__QuestDb__RetentionDays: "${OTOPCUA_HISTORIAN_QUESTDB_RETENTION_DAYS:-}"
|
||||
# Continuous historization — see central-1 for the full note + the gate recipe. Both centrals
|
||||
# carry it so the pair is symmetric; :9200 round-robins them, and the recorder must exist on
|
||||
# whichever node holds the drivers.
|
||||
|
||||
@@ -153,6 +153,19 @@ Ordering follows from that: the drain reads in `enqueued_at_utc` order (with `id
|
||||
as a tiebreak so the ordering is total), because a hashed key carries no
|
||||
insertion sequence.
|
||||
|
||||
Gateway delivery carries a separate nonzero `DeliveryId` GUID. Direct event instances
|
||||
create it once; retries and record copies preserve it. It is excluded from queue JSON
|
||||
and business equality, so the existing payload fingerprint and pair deduplication stay
|
||||
unchanged. The drain derives its delivery GUID solely from the persisted row `id`,
|
||||
including migrated legacy rows. It never hashes a reconstructed payload or mints an ID
|
||||
per send. The versioned encoding in `AlarmDeliveryIds` therefore survives retry,
|
||||
restart, payload edits and replicated failover without a schema migration.
|
||||
|
||||
Different business payloads at the same timestamp remain distinct; completely identical
|
||||
payloads deliberately converge. An explicitly empty direct delivery ID is rejected.
|
||||
The Gateway mapper sends this GUID as the wire event ID; the live alarm readback test
|
||||
requires that exact ID. SQL Server and QuestDB retain their existing event-ID schemes.
|
||||
|
||||
`EnqueueAsync` does a single `INSERT … ON CONFLICT DO NOTHING` on the hot path.
|
||||
To avoid a `SELECT COUNT(*)` on every enqueue, the sink keeps an in-memory
|
||||
non-dead-lettered row counter (seeded at startup, kept current by every mutation,
|
||||
|
||||
+15
-4
@@ -71,10 +71,10 @@ same condition is wired as an **event notifier of every referencing equipment fo
|
||||
reconnect snapshot entries, OpcUaClient's first event per condition after a start or deploy respawn — flags
|
||||
`AlarmEventArgs.IsRestatement` (optional, default `false`; FOCAS, MTConnect and live Galaxy transitions never
|
||||
set it; AbCip ALMA sets it for the drop-out update below). The flag governs **history only, never node state**: the projector still projects, but a flagged `Retrigger`
|
||||
on a condition already Active+unacked with the **same message AND severity**, or a flagged `Clear` on an
|
||||
already-inactive condition, writes no `/alerts` row; so does a flagged `Unspecified` update on an already-active
|
||||
condition, which moves the node's message/severity (one condition event, active/acked kept) — AbCip ALMA's
|
||||
governing limit dropping out (HH→H), and OpcUaClient's first sight of an inactive condition after a respawn. Before this every gateway reconnect wrote a duplicate
|
||||
on a condition already Active+unacked with the **same message AND severity**, a flagged `Clear` on an
|
||||
already-inactive condition, or a flagged `Acknowledge` on an already-acknowledged one (#657), writes no `/alerts` row;
|
||||
so does a flagged `Unspecified` update on an already-active condition, which moves the node's message/severity (one
|
||||
condition event, active/acked kept) — AbCip ALMA's governing limit dropping out (HH→H). Before this every gateway reconnect wrote a duplicate
|
||||
"Activated" (and "Cleared") row per alarm while clients saw no event. A **live** `Retrigger` is a genuine
|
||||
re-occurrence and keeps its row; a changed severity (an MTConnect Warning→Fault that keeps its text) does too.
|
||||
Residual: a Galaxy alarm that genuinely re-occurred during an outage and is still Active+unacked with identical
|
||||
@@ -88,6 +88,17 @@ same condition is wired as an **event notifier of every referencing equipment fo
|
||||
`Active = false` on a cold host or an inactive prior, so an alarm active in AVEVA projected **inactive** and was
|
||||
invisible to clients. OpcUaClient emits it for inactive → (active, acked) instead of `Raise` + `Acknowledge`,
|
||||
which showed clients an Active+**unacked** intermediate event.
|
||||
- **`AlarmTransitionKind.InactiveUnacknowledged` (#657)** — its mirror: "inactive AND unacknowledged" (went active and
|
||||
cleared unacked) in ONE projection, never `Raise`/`Retrigger` + `Clear` (which showed clients the condition Active
|
||||
again for an instant). Rows only for what changed, flag or not: "Activated" (raise time when known) + "Cleared" when
|
||||
the condition was acknowledged (an un-ack needs a new occurrence), "Cleared" alone when it was Active+unacked, none
|
||||
when it already showed inactive+unacked. OpcUaClient emits it for every move into inactive+unacked except from
|
||||
Active+unacked (a plain `Clear`). Its **first sight** of an inactive condition (after a start or deploy respawn, when
|
||||
the host may still hold ANY state) is `InactiveUnacknowledged`, or `Clear` + `Acknowledge` for inactive+acked — both
|
||||
flagged — so a host already showing that state writes nothing, and a host that retained the condition Active gets a
|
||||
real "Cleared". Pre-#657 a respawn wrote a duplicate "Activated" + "Cleared" pair per inactive+unacked condition, and
|
||||
an inactive+acked first sight went out as `Unspecified` (an "Activated" row for an inactive alarm; a retained-Active
|
||||
condition was never cleared). Residual (pre-existing, not a regression): a cold host treats an unseen condition as acked, so after a node restart each upstream inactive+unacked condition still writes "Activated" + "Cleared" once. E2E: `Runtime.Tests/NativeAlarmRestatementEndToEndTests`.
|
||||
|
||||
### Native condition state survives a full rebuild (Gitea #641)
|
||||
|
||||
|
||||
+69
-2
@@ -57,6 +57,11 @@ otopcua-cli write -u opc.tcp://localhost:4840 -n "ns=2;s=MyNode" -v 42 -U operat
|
||||
|
||||
### Failover
|
||||
|
||||
**Mapped ports and internal hostnames.** Endpoint discovery rewrites the host *and port* of the endpoint the server
|
||||
advertises to the URL you passed (when the scheme matches), so connecting through a port mapping works even though the
|
||||
server advertises its internal address (for example the docker-dev rig: `-u opc.tcp://localhost:4841` reaches the
|
||||
node mapped to 4841, although every container advertises `:4840`).
|
||||
|
||||
When `-F` is provided, the shared service tries the primary URL first, then each failover URL in order. For long-running commands (`subscribe`, `alarms`), the service monitors the session via keep-alive and automatically reconnects to the next available server on failure.
|
||||
|
||||
```bash
|
||||
@@ -205,21 +210,78 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z"
|
||||
|
||||
# Aggregate: 1-hour average
|
||||
# Aggregate: 1-hour average (every backend; on Gateway it is computed from raw samples since #681 —
|
||||
# capped by ServerHistorian:MaxTimeWeightedRawRows, so read long dense windows in slices)
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
-n "ns=1;s=TestMachine_001.TestHistoryValue" \
|
||||
--start "2026-03-25T00:00:00Z" --end "2026-03-30T00:00:00Z" \
|
||||
--aggregate Average --interval 3600000
|
||||
|
||||
# Modified-value history (what HistoryUpdate changed): prior value, update type, user, modification time
|
||||
otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa -U apikey -P otua_<keyId>_<secret> \
|
||||
-n "ns=2;s=Plant/Modbus/dev1/Speed" \
|
||||
--start "2026-01-15T00:00:00Z" --end "2026-01-16T00:00:00Z" --modified
|
||||
```
|
||||
|
||||
`--modified` sends `ReadRawModifiedDetails` with `IsReadModified = true` (OPC UA Part 11, Gitea #668) and prints one line per
|
||||
recorded modification: the source timestamp, the **prior** value (`(none)` for an insert, which had none), its status, the update type
|
||||
(`Insert` / `Replace` / `Update` / `Delete`), the user (the API key's principal, or `anonymous`) and the modification time. It follows
|
||||
continuation points up to `--max`. A server on the `Gateway` backend (no modification trail) answers `BadHistoryOperationUnsupported`.
|
||||
`--modified` cannot be combined with `--aggregate`. See [Historian.md §Modified-value history](Historian.md#modified-value-history-part-11).
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `-n` / `--node` | Node ID to read history for (required) |
|
||||
| `--start` | Start time in ISO 8601 UTC format, e.g. `2026-01-15T08:00:00Z` (default: 24 hours ago) |
|
||||
| `--end` | End time in ISO 8601 UTC format, e.g. `2026-01-15T09:00:00Z` (default: now) |
|
||||
| `--max` | Maximum number of values (default: 1000) |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation |
|
||||
| `--aggregate` | Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative |
|
||||
| `--interval` | Processing interval in milliseconds for aggregates (default: 3600000) |
|
||||
| `--modified` | Read modified-value history (Part 11 `IsReadModified`) instead of raw samples |
|
||||
|
||||
### historyupdate
|
||||
|
||||
Inserts, replaces, updates or deletes stored history on a historized node via the OPC UA `HistoryUpdate`
|
||||
service (requires a server whose historian backend supports updates and a session holding the history-write
|
||||
permission). Prints one line per item (`index time StatusCodeName`; a code the SDK has no name for prints as
|
||||
`0xXXXXXXXX`) followed by the overall service result. It exits non-zero unless **every** result is Good: a Bad
|
||||
**or Uncertain** item, a non-Good node or service result, or a Good service result that is missing per-item
|
||||
results (fewer than the items sent, printed as `(no result returned)`) all fail the command.
|
||||
|
||||
```bash
|
||||
# Insert two values
|
||||
otopcua-cli historyupdate -u opc.tcp://localhost:4840/OtOpcUa -n "ns=2;s=Plant/Modbus/dev1/Speed" \
|
||||
--mode insert --value 1.5 --time 2026-01-15T08:00:00Z --value 2.5 --time 2026-01-15T09:00:00Z
|
||||
|
||||
# Replace an int64 value
|
||||
otopcua-cli historyupdate -u ... -n "..." --mode replace --type int64 --value 42 --time 2026-01-15T08:00:00Z
|
||||
|
||||
# Delete a raw range / specific timestamps
|
||||
otopcua-cli historyupdate -u ... -n "..." --mode delete-raw --start 2026-01-15T08:00:00Z --end 2026-01-15T10:00:00Z
|
||||
otopcua-cli historyupdate -u ... -n "..." --mode delete-at-time --time 2026-01-15T08:00:00Z
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `-n` / `--node` | Node ID to modify (required) |
|
||||
| `--mode` | `insert`, `replace`, `update`, `delete-raw` or `delete-at-time` (required) |
|
||||
| `--value` | Value to write; repeatable, paired positionally with `--time` (insert/replace/update) |
|
||||
| `--time` | ISO 8601 UTC timestamp; repeatable (insert/replace/update/delete-at-time) |
|
||||
| `--type` | `double` (default), `int64`, `bool`, `string`, `datetime` - how `--value` is parsed |
|
||||
| `--start` / `--end` | Range for `delete-raw` (both required; `--end` later than `--start`) |
|
||||
|
||||
Expected per-item codes on the SQL Server historian backend: `insert` ⇒ `GoodEntryInserted` (repeat ⇒ `BadEntryExists`), `replace` ⇒ `GoodEntryReplaced`
|
||||
(`BadNoEntryExists` if nothing is stored at that time), `update` ⇒ either Good code, `delete-at-time` ⇒ `Good` / `BadNoEntryExists`; `delete-raw` reports one
|
||||
node-level code (`Good`, or `BadNoData` if the range held nothing). A node that is not historized, or a server on the Gateway historian backend (no
|
||||
updater, so no `HistoryWrite` access bit), answers `BadHistoryOperationUnsupported`; a `NaN`/`Infinity` double is `BadInvalidArgument`. The session's API key
|
||||
(`-U apikey -P otua_<keyId>_<secret>`) needs the `HistoryUpdate` permission — via a `Cluster`-scoped grant when `Security:Acl:Enabled` is off, or a scoped
|
||||
grant on the node when it is on; a key with only `WriteOperate` gets `BadUserAccessDenied`. See [Historian.md §HistoryUpdate](Historian.md#historyupdate) and
|
||||
[security.md](security.md#historyupdate-historian-writes). `historyread --aggregate` only knows the twelve names in the table below; the SQL backend's other
|
||||
aggregates (Range, Delta, PercentGood, …) need a different client. A refused history read (for example
|
||||
`BadHistoryOperationUnsupported`) prints one error line and exits non-zero; it does not dump a stack trace.
|
||||
|
||||
`delete-raw` is sent with `IsDeleteModified=false`. Invalid or inconsistent options (unknown mode, unpaired
|
||||
value/time, unparsable value, options that do not apply to the mode) fail before any connection is made.
|
||||
|
||||
#### Aggregate mapping
|
||||
|
||||
@@ -232,6 +294,11 @@ otopcua-cli historyread -u opc.tcp://localhost:4840/OtOpcUa \
|
||||
| `Start` | `first` | `AggregateFunction_Start` |
|
||||
| `End` | `last` | `AggregateFunction_End` |
|
||||
| `StandardDeviation` | `stddev`, `stdev` | `AggregateFunction_StandardDeviationPopulation` |
|
||||
| `TimeAverage` | `timeavg` | `AggregateFunction_TimeAverage` |
|
||||
| `TimeAverage2` | `timeavg2` | `AggregateFunction_TimeAverage2` |
|
||||
| `Total` | | `AggregateFunction_Total` |
|
||||
| `Total2` | | `AggregateFunction_Total2` |
|
||||
| `Interpolative` | `interp` | `AggregateFunction_Interpolative` |
|
||||
|
||||
### alarms
|
||||
|
||||
|
||||
+1
-1
@@ -206,7 +206,7 @@ All times are in UTC. Invalid input turns red on blur.
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation |
|
||||
| Aggregate | Raw (default), Average, Minimum, Maximum, Count, Start, End, Standard Deviation, Time Average, Time Average 2, Total, Total 2, Interpolative |
|
||||
| Interval (ms) | Processing interval for aggregate queries (shown only for aggregates) |
|
||||
| Max Values | Maximum number of raw values to return (default 1000) |
|
||||
|
||||
|
||||
+72
-7
@@ -22,14 +22,30 @@ Two related concerns get their own dedicated pages and are **only summarised + l
|
||||
|
||||
The Host (`Program.cs`) loads `appsettings.json`, then overlays a **per-role** file chosen from the cluster roles:
|
||||
|
||||
- A single role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (roles joined with `-`, ordinal-sorted).
|
||||
- A single plain role → `appsettings.{role}.json` (e.g. `appsettings.driver.json`, `appsettings.admin.json`).
|
||||
- Both roles → `appsettings.admin-driver.json` (plain roles joined with `-`, ordinal-sorted).
|
||||
- `cluster-{ClusterId}` roles are **ignored** when naming the overlay (#678.6) — `[driver, cluster-SITE-A]` loads `appsettings.driver.json`.
|
||||
- `appsettings.{ASPNETCORE_ENVIRONMENT}.json` (e.g. `appsettings.Development.json`) is layered on by the host builder.
|
||||
|
||||
All role overlays are **optional** — the base `appsettings.json` plus the Options-class C# defaults are enough to boot. The roles themselves come from `EffectiveRoles.Resolve` (`ZB.MOM.WW.OtOpcUa.Cluster`) — `Cluster:Roles`, falling back to `OTOPCUA_ROLES` when that is empty (see [`ServiceHosting.md`](ServiceHosting.md) and the table below).
|
||||
|
||||
The checked-in `appsettings*.json` files are deliberately thin: they carry only `Serilog` and the `Security:Ldap` overlay. Everything else (`OpcUa`, `Cluster`, `ConnectionStrings`/`ConfigDb`) binds from the Options-class defaults documented below unless an operator adds the section explicitly or supplies the corresponding environment variable.
|
||||
|
||||
<a id="role-overlay-audit"></a>
|
||||
#### Role overlay audit (#678.6)
|
||||
|
||||
Until #678.6 the overlay never loaded on a node carrying a `cluster-*` role (the cluster role leaked into the filename). Fixing that **activates** the shipped overlays on those nodes (the docker-dev rig and any per-cluster-mesh production node), so each key was audited against base `appsettings.json` / the Options defaults. Env vars and command-line args are re-appended after the overlay, so any value a deployment sets explicitly (the rig sets every `Security__Ldap__*` key) still wins.
|
||||
|
||||
| Key | Overlay(s) | Base / default | Effect on activation |
|
||||
|---|---|---|---|
|
||||
| `Serilog:MinimumLevel:Default` = `Information` | all three | Serilog default `Information` | no-op (in `Development` it outranks `appsettings.Development.json`'s `Debug`, by design) |
|
||||
| `…Override:Akka` = `Information` | all three | inherits `Default` (`Information`) | no-op |
|
||||
| `…Override:Microsoft.AspNetCore` = `Warning` | `admin`, `admin-driver` | unset (`Information`) | quieter ASP.NET Core logs; log-level only, matches what the rig already pins |
|
||||
| `…Override:Opc.Ua` = `Information` | `admin-driver`, `driver` | inherits `Default` | no-op. **`driver` shipped `Debug` — changed to `Information` in #678.6**: activating a Debug OPC UA stack trace on every production driver node would flood Serilog (the same pipeline starvation the rig's EF/AspNetCore pins guard against). Raise it per node with `Serilog__MinimumLevel__Override__Opc.Ua=Debug`. |
|
||||
| `Security:Ldap:DevStubMode` = `false` | all three | `false` | no-op; the overlay's point is that it outranks a developer's `appsettings.Development.json` (`true`) |
|
||||
| `Security:Ldap:Transport` = `Ldaps` | all three | `LdapOptions.Transport` default `None` | **behaviour-changing, kept deliberately** (documented secure production default). A cluster-role node that configures `Security:Ldap:Server`/`Port` for a plaintext LDAP but never sets `Transport` would now dial LDAPS and fail AdminUI sign-in; set `Security__Ldap__Transport` (the rig already sets `None` explicitly). Data-plane auth is API-key based and unaffected; driver-only nodes have no AdminUI. |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## `appsettings` sections
|
||||
@@ -38,7 +54,7 @@ The checked-in `appsettings*.json` files are deliberately thin: they carry only
|
||||
|
||||
- **Purpose:** logging. Console + rolling daily file sink, layered with the shared `ZB.MOM.WW.Telemetry` enrichers (`AddZbSerilog` in `Program.cs`).
|
||||
- **Where bound:** `builder.AddZbSerilog(...)` reads `Serilog` from configuration (`ReadFrom.Configuration`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Debug`, `Akka: Information`).
|
||||
- **Checked-in shape** (`appsettings.json`): `Using` = `[ "Serilog.Sinks.Console", "Serilog.Sinks.File" ]`, `WriteTo` = a `Console` sink and a `File` sink (`path: logs/otopcua-.log`, `rollingInterval: Day`). Role overlays add `MinimumLevel` / `Override` blocks (e.g. `Opc.Ua: Information`, `Akka: Information`).
|
||||
|
||||
### `OpcUa`
|
||||
|
||||
@@ -377,9 +393,14 @@ The Galaxy/MxAccess connection settings are **not an `appsettings` section.** Th
|
||||
|
||||
> The `OTOPCUA_GALAXY_*` environment variables that v1's in-process `Galaxy.Host` consumed **no longer live in this repo** — they moved into the separately-installed mxaccessgw gateway's own config (see the v1 archive pointer in `docs/README.md` and the Galaxy overview at [`docs/drivers/Galaxy.md`](drivers/Galaxy.md)). The only Galaxy connection secret this repo touches is the gateway API key via `ApiKeySecretRef` above.
|
||||
|
||||
### Historian config (HistorianGateway)
|
||||
### Historian config (`ServerHistorian:Backend` — HistorianGateway, SQL Server or QuestDB)
|
||||
|
||||
The historian backend is the external **`ZB.MOM.WW.HistorianGateway`** sidecar, consumed as the
|
||||
The historian backend is **pluggable**, selected by `ServerHistorian:Backend` (case-insensitive; default `Gateway`;
|
||||
unknown value ⇒ host start fails listing the allowed values): `Gateway` is the external **`ZB.MOM.WW.HistorianGateway`**
|
||||
sidecar below, `SqlServer` is a plain SQL Server database (see [`ServerHistorian:SqlServer`](#serverhistoriansqlserver-backendsqlserver)),
|
||||
and `QuestDb` is a QuestDB time-series database reached over the Postgres wire protocol (see
|
||||
[`ServerHistorian:QuestDb`](#serverhistorianquestdb-backendquestdb)).
|
||||
The default backend is the gateway, consumed as the
|
||||
`ZB.MOM.WW.HistorianGateway.Client` gRPC package (the retired Wonderware TCP sidecar is documented at
|
||||
[`docs/drivers/Historian.Wonderware.md`](drivers/Historian.Wonderware.md)). The OtOpcUa host reads three
|
||||
appsettings sections — `ServerHistorian` (read path + gateway connection), `ContinuousHistorization`
|
||||
@@ -394,9 +415,49 @@ key must carry the scopes `historian:read`, `historian:write`, `historian:tags:w
|
||||
[`docs/Historian.md`](Historian.md) for the full key reference, the migration note (old Wonderware keys →
|
||||
gateway keys), and the deployment prerequisites.
|
||||
|
||||
#### `ServerHistorian:SqlServer` (`Backend=SqlServer`)
|
||||
|
||||
Read only when `ServerHistorian:Backend` is `SqlServer`; the gateway keys (`Endpoint`, `ApiKey`, `UseTls`, …) are then not required.
|
||||
Validated at host start (driver-role nodes) when `ServerHistorian:Enabled` **or** `AlarmHistorian:Enabled` is true.
|
||||
|
||||
| Key | Default | Validation / meaning |
|
||||
|---|---|---|
|
||||
| `ServerHistorian:Backend` | `Gateway` | `Gateway`, `SqlServer` or `QuestDb`; anything else fails host start (checked even when the historian is disabled). |
|
||||
| `ServerHistorian:SqlServer:ConnectionString` | `""` | **Required.** Historian database connection string (not the ConfigDb). Supply via `${secret:OT-HISTORIAN-SQL}` or env `ServerHistorian__SqlServer__ConnectionString`; the database must already exist. |
|
||||
| `ServerHistorian:SqlServer:Schema` | `hist` | Must match `^[A-Za-z_][A-Za-z0-9_]{0,127}$` (spliced into DDL/DML). |
|
||||
| `ServerHistorian:SqlServer:CommandTimeout` | `00:00:30` | Must be `> 0`; rounded up to whole seconds. |
|
||||
| `ServerHistorian:SqlServer:AutoCreateSchema` | `true` | `true` applies the embedded schema scripts under an app-lock on first use (upgrading a v1 schema to v2); `false` only verifies `SchemaVersion >= 1` and fails every historian call (health `LastError`, no crash) if behind. A verified **v1** schema (no `002_sample_modified.sql`) is served with modified-value reads and HistoryUpdate refused `BadHistoryOperationUnsupported`, retention skipping `SampleModified`, and one Warning per boot (#674). |
|
||||
| `ServerHistorian:SqlServer:RetentionDays` | unset | Days of history kept (source time), `>= 1` when set; unset keeps everything. Enables the Primary-gated purge of `hist.Sample`/`hist.Event` (see [Historian.md](Historian.md#retention-retentiondays)). |
|
||||
| `ServerHistorian:SqlServer:PurgeInterval` | `01:00:00` | Purge cadence; must be `> 0`. |
|
||||
| `ServerHistorian:SqlServer:PurgeBatchSize` | `4000` | Rows per `DELETE TOP (n)` batch; `>= 1`. Keep under ~5000 to avoid lock escalation. |
|
||||
|
||||
#### `ServerHistorian:QuestDb` (`Backend=QuestDb`)
|
||||
|
||||
Read only when `ServerHistorian:Backend` is `QuestDb`; the gateway keys are then not required. Validated at host start (driver-role nodes)
|
||||
when `ServerHistorian:Enabled` **or** `AlarmHistorian:Enabled` is true (`QuestDbHistorianOptions.Validate()`).
|
||||
|
||||
| Key | Default | Validation / meaning |
|
||||
|---|---|---|
|
||||
| `ServerHistorian:QuestDb:ConnectionString` | `""` | **Required.** Npgsql connection string for QuestDB's PG-wire port, e.g. `Host=questdb;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading`. Env `ServerHistorian__QuestDb__ConnectionString` / `${secret:}` — never commit a real one. When the string does not set Npgsql `Maximum Pool Size`, the backend caps it at **24** (Npgsql's own default of 100 exceeds QuestDB's 64-connection default limit); an explicit value is kept as written, and the effective value is logged at Information. Keep the per-node total of a pair under QuestDB's limit. |
|
||||
| `ServerHistorian:QuestDb:TablePrefix` | `hist_` | Must match `^[A-Za-z0-9_]+$` (spliced into DDL/DML). Tables are `{prefix}sample`, `{prefix}event`, `{prefix}tag`. |
|
||||
| `ServerHistorian:QuestDb:RetentionDays` | unset | `>= 1` when set. Applied as table TTL to the sample and event tables (enforced on write commits only — a quiet table never drops). Unset keeps data forever. |
|
||||
| `ServerHistorian:QuestDb:CommandTimeout` | `00:00:30` | Must be `> 0`; rounded up to whole seconds. |
|
||||
| `ServerHistorian:QuestDb:AutoCreateSchema` | `true` | `true` creates the tables and reconciles the TTL on first use; `false` only verifies the tables exist and fails every historian call (health `LastError`, no crash) if one is missing. |
|
||||
| `ServerHistorian:QuestDb:VisibilityTimeout` | `00:00:05` | Must be `>= 0`. How long a write waits for its rows to become visible to queries (WAL apply); `0` disables the wait. A timeout only logs a Warning. |
|
||||
|
||||
HistoryUpdate (SqlServer and QuestDb only; QuestDb deletes are soft — tombstones) additionally needs `ServerHistorian:Enabled=true` and a key
|
||||
whose principal holds the `HistoryUpdate` permission — see [`docs/security.md`](security.md#historyupdate-historian-writes). It is served
|
||||
only on the redundancy **Primary** (`BadInvalidState` elsewhere, #674) and is bounded by one backend-independent key:
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `ServerHistorian:HistoryUpdateMaxFutureSkew` | `1.00:00:00` (1 day) | TimeSpan. A HistoryUpdate (Insert / Replace / Update) item whose `SourceTimestamp` is later than server-clock now + this is refused `BadOutOfRange` before any backend sees it (#674 — a far-future typo could otherwise age out a QuestDB TTL table or poison the history). Applies to every backend; the live recorder is not bounded. Non-positive disables the bound. | Full semantics,
|
||||
schema, aggregates and the docker-dev recipe: [`docs/Historian.md`](Historian.md#sql-server-backend) (SqlServer) and
|
||||
[`docs/Historian.md`](Historian.md#questdb-backend) (QuestDb: the µs timestamp rules, pre-1970 limits, TTL and the docker-dev overlay).
|
||||
|
||||
#### `ContinuousHistorization` (recorder + outbox knobs)
|
||||
|
||||
Only the knobs; the gateway connection comes from `ServerHistorian`. Full descriptions in
|
||||
Only the knobs; the backend connection comes from `ServerHistorian`. Outbox records are format v2 — **drain or delete `OutboxPath` before downgrading a node** to a pre-v2 build (see [`docs/Historian.md` §Outbox format v2 and rollback](Historian.md#outbox-format-v2-and-rollback)); undecodable records are skipped and counted by the gauge `otopcua.historian.outbox.unreadable`. Full descriptions in
|
||||
[`docs/Historian.md` §ContinuousHistorization configuration](Historian.md#continuoushistorization-configuration).
|
||||
|
||||
| Key | Default | Meaning |
|
||||
@@ -410,6 +471,7 @@ Only the knobs; the gateway connection comes from `ServerHistorian`. Full descri
|
||||
| `ContinuousHistorization:DrainConcurrency` | `8` | Historian tags one drain pass writes **concurrently** — a batch is grouped per tag and each group is its own gateway round-trip, so this turns a sequential `tags × latency` pass into `⌈tags / concurrency⌉ × latency` while bounding outstanding round-trips. `1` = fully sequential; `< 1` is clamped to 1 with a `Validate()` warning. The ack is unchanged (all-or-nothing after every group reports), as is the #538 quarantine rule. |
|
||||
| `ContinuousHistorization:Capacity` | `500000` | Max un-acked entries before drop-oldest; `0` = unbounded. The #538 disk backstop for a historian refusing **every** write — the one case the per-tag quarantine cannot resolve. |
|
||||
| `ContinuousHistorization:MaxWriteAttempts` | `5` | #538 poison-tag budget: consecutive failed passes one tag may accumulate — **counted only across passes where *other* tags wrote successfully** — before its entries are abandoned so the FIFO head can advance. |
|
||||
| `ContinuousHistorization:CaptureWhilePrimaryUnreachable` | `true` | #672: a standby stood down for its mesh-partner Primary (#537) resumes capture as soon as the failure detector reports that Primary **unreachable**, rather than after auto-down + a fresh snapshot — a crash-failover gap of ~25–30 s becomes the failure-detector window (~10–12 s). Cost: duplicate capture while a live Primary is merely unreachable and under a real partition (absorbed idempotently on SqlServer/QuestDB only for source-timestamped drivers; Modbus-style poll-time stamps and the Gateway backend store duplicates). Affects only the capture verdict. `false` = pre-#672 behaviour. |
|
||||
| `ContinuousHistorization:MinBackoffSeconds` / `MaxBackoffSeconds` | `1` / `30` | Initial and capped exponential retry backoff after a failed drain pass. |
|
||||
|
||||
### `LocalDb` (pair-local config cache — driver role only)
|
||||
@@ -498,7 +560,10 @@ historian secret this repo reads from the environment is the gateway API key:
|
||||
|
||||
| Variable | Effect / default |
|
||||
|---|---|
|
||||
| `ServerHistorian__ApiKey` | The HistorianGateway peppered-HMAC key (`histgw_<id>_<secret>`) sent as `Authorization: Bearer`. Supply via environment — **never commit**. Required when `ServerHistorian:Enabled=true`. |
|
||||
| `ServerHistorian__ApiKey` | The HistorianGateway peppered-HMAC key (`histgw_<id>_<secret>`) sent as `Authorization: Bearer`. Supply via environment — **never commit**. Required when `ServerHistorian:Enabled=true` **and `Backend=Gateway`**. |
|
||||
| `ServerHistorian__Backend` | `Gateway` (default), `SqlServer` or `QuestDb`. |
|
||||
| `ServerHistorian__SqlServer__ConnectionString` | The SQL Server historian connection string. Required when `Backend=SqlServer` and the historian (or alarm history) is enabled. Supply via environment / `${secret:}` — **never commit**. |
|
||||
| `ServerHistorian__QuestDb__ConnectionString` | The QuestDB historian Npgsql connection string. Required when `Backend=QuestDb` and the historian (or alarm history) is enabled. Supply via environment / `${secret:}` — **never commit**. |
|
||||
|
||||
### Driver integration-test / fixture sim endpoints
|
||||
|
||||
|
||||
+65
-5
@@ -242,6 +242,28 @@ Things to know before touching any of this:
|
||||
re-subscribing** so a filtering driver does not accumulate two overlapping ones. There is still
|
||||
deliberately no unsubscribe-on-**empty**: a removed alarm tag's condition node is torn down on the
|
||||
address-space rebuild and `ForwardNativeAlarm` drops any transition whose `ConditionId` no longer maps.
|
||||
- ⚠️ **A data re-subscribe must not touch the alarm feed (#662).** Every deploy's bulk push re-sends
|
||||
`SetDesiredSubscriptions` to every kept driver, and `HandleSubscribeAsync` drops the prior DATA
|
||||
subscription first. That teardown used to be the full `DetachSubscription`, which also unhooked
|
||||
`OnAlarmEvent` and forgot the alarm handle — so on any deploy that did **not** respawn a driver (e.g. one
|
||||
that only disabled another driver) its alarm subscription kept logging "established" while every event
|
||||
reached nobody, until a reconnect or respawn. The data path now uses `DetachDataSubscription`; only a lost
|
||||
connection (`DisconnectObserved` / `ForceReconnect`) and `PostStop` detach both. The **#488 periodic
|
||||
reconcile covers alarms too** (`ReconcileAlarmSubscription`): it re-attaches a missing `OnAlarmEvent`
|
||||
handler and re-subscribes when the live alarm handle does not match the desired set, so an alarm subscribe
|
||||
that failed on (re)connect — a session not yet ready after an outage — is retried every health-poll tick
|
||||
instead of gating the feed for the life of the connection. OpcUaClient removes a half-built alarm
|
||||
subscription from its session when the subscribe throws, so those retries do not accumulate.
|
||||
End-to-end pin: `NativeAlarmFeedSurvivesDeployTests`.
|
||||
- **There is no in-place config path — a config change is a respawn (#665).** Since #516 any DriverConfig,
|
||||
DriverType or ResilienceConfig change is stop + respawn, so the factory is the single parse authority. The old
|
||||
in-place seam (host `ApplyChildDelta` → actor `ApplyDelta` → `ReinitializeAsync`, plus the mid-connect
|
||||
`AdoptConfigDuringInit`, `ApplyResult` and `DeltaApplied`) had no producer and was deleted after two latent
|
||||
bugs were found in it: a Connected reinit kept subscription handles the driver had dropped (#663), and the
|
||||
mid-connect path re-ran `InitializeAsync(newConfig)` on two drivers that ignored the JSON (#664). Kept from that
|
||||
work: Calculation's `InitializeAsync` adopts a config with a body and Galaxy's refuses a non-equivalent one,
|
||||
because the `IDriver` contract is that a driver serves the document it is handed. `ReinitializeAsync` stays as a
|
||||
per-driver contract the runtime does not call.
|
||||
- **Restart tests must assert a FUNCTION, never existence.** `RestartDriver_respawns_the_child_without_an_actor_name_collision`
|
||||
asserts only that a child exists after a restart, and it passed green against the dead driver 03/S16
|
||||
describes. It is kept — its own subject, the `InvalidActorNameException`, is real — but the load-bearing
|
||||
@@ -400,7 +422,10 @@ Two consequences worth knowing:
|
||||
- **`CalculationDriver` can never trip it, correctly.** It reports `Healthy` whenever it is running and has
|
||||
no transport to be degraded about. Every other driver reaches a non-Healthy state on sustained failure:
|
||||
the nine poll drivers via `Degraded`, MQTT via its live connection state, Galaxy via its reconnect
|
||||
supervisor. This was surveyed across all 12 before relying on it — it is the assumption the whole
|
||||
supervisor, OpcUaClient via a bad session keep-alive (`Degraded`, since Gitea #670 — before that its health
|
||||
only moved on `ReadAsync`, which a subscription-only driver never calls, so it stayed `Healthy` through any
|
||||
outage and this survey was wrong about it; good keep-alives and notifications now advance its read stamp, so
|
||||
an idle-but-alive subscription does not look dead either). This was surveyed across all 12 before relying on it — it is the assumption the whole
|
||||
mechanism rests on.
|
||||
- **A device that accepts a connection but serves no data will cycle**, reconnecting once per threshold.
|
||||
That is the correct reading of a broken device, it is rate-limited, and it is visible in the health
|
||||
@@ -416,6 +441,40 @@ Two consequences worth knowing:
|
||||
**notices**, so a silent-success termination path — a stream that ends cleanly, a poll loop that exits
|
||||
without throwing — bypasses it entirely.
|
||||
|
||||
**A driver that KNOWS its connection is gone can skip the wait: `IDriverConnectionLossSource` (Gitea #676).** The
|
||||
watchdog has to prove a loss from silence, so it costs ~90 s even when the protocol already said "this connection
|
||||
will never come back". An opt-in `Core.Abstractions` interface lets a driver say so:
|
||||
`event EventHandler<DriverConnectionLossEventArgs> ConnectionLost` (a reason string). `DriverInstanceActor` attaches
|
||||
in `PreStart` and detaches in `PostStop` — the same lifetime as `IRediscoverable`, for the same reason (the `IDriver`
|
||||
outlives the actor on a respawn) — marshals each raise to the actor thread (`Self.Tell`; actor state is never touched
|
||||
on the driver's thread), and acts on it **only while `Connected`**, through the very handler the watchdog's
|
||||
`DisconnectObserved` uses (detach, record a fault, `Reconnecting`, teardown-then-init on the retry). In
|
||||
`Connecting`/`Reconnecting`/`Stubbed` it is ignored, not deferred: a re-initialise is already pending, and acting on a
|
||||
stale raise later could tear down the connection that attempt is building. The two are complementary, not
|
||||
alternatives:
|
||||
|
||||
- **The signal is for DEFINITIVE losses only.** A driver must raise only when no in-place recovery is possible, at most
|
||||
once per connection, re-armed by a re-initialise. An ambiguous failure — a timeout, a stalled socket, "no
|
||||
communication" — must not raise: a stalled link that resumes keeps its session, subscriptions and in-place recovery,
|
||||
and tearing it down would cost all three. Those stay with the watchdog, whose conjunction is the honest test for them.
|
||||
- **Today only `OpcUaClientDriver` implements it:** keep-alive `BadSessionIdInvalid` / `BadSessionClosed` /
|
||||
`BadSessionNotActivated` / `BadServerHalted`; a CLOSED connection (`BadSecureChannelClosed` /
|
||||
`BadConnectionClosed`) once sustained — the second closed-channel keep-alive in one outage, which is what a real
|
||||
process stop actually produces (live gate A1), and the SDK never reopens a closed connection by itself; or a
|
||||
subscription the server REPORTED dropped (not the driver's
|
||||
publishing-stall heuristic, which is an inference from silence and stays with the watchdog; see
|
||||
[OpcUaClient.md](drivers/OpcUaClient.md) §"Upstream outage and recovery" for why `BadServerHalted` is on the list —
|
||||
it was measured, the SDK never re-opens a halted channel). A restarted, stopped or killed upstream therefore
|
||||
recovers in about one keep-alive plus one reconnect interval instead of ~100 s; a stall/pause
|
||||
(`BadRequestTimeout`/`BadNoCommunication`) still goes through the watchdog and keeps its in-place recovery.
|
||||
- ⚠️ **Known gap:** "only while `Connected`" plus "once per connection" means a raise the host receives while not yet
|
||||
Connected (e.g. the protocol proves the fresh connection dead before `InitializeSucceeded` is processed) is swallowed,
|
||||
and that connection falls back to the watchdog — the pre-#676 recovery time, in a narrow window.
|
||||
- Pinned by `DriverInstanceActorConnectionLossSignalTests` (Reconnecting within seconds while the watchdog is set to an
|
||||
hour; a falsification control with no raise that stays Connected; a raise mid-connect ignored; a stray watchdog
|
||||
`DisconnectObserved` in `Reconnecting` is a no-op; detach on stop) and `OpcUaClientConnectionLossSignalTests`
|
||||
(definitive raises once, ambiguous and the stall heuristic never, re-armed after re-init).
|
||||
|
||||
**The reconnect retry now tears down before re-initialising.** `Reconnecting`'s `RetryConnect` calls
|
||||
`ShutdownAsync` before `InitializeAsync` — the sequence every driver's own `ReinitializeAsync` already
|
||||
uses — because several drivers **overwrite** an `IAsyncDisposable` transport field inside `InitializeAsync`
|
||||
@@ -426,8 +485,8 @@ have leaked ~360 transports at the 10 s retry interval. The shutdown is best-eff
|
||||
on it still gets its initialise attempt.
|
||||
|
||||
**`InitializeAsync` itself releases the previous device/connection state first (#630).** The retry above is not
|
||||
the only way in: `DriverInstanceActor.AdoptConfigDuringInit` calls `InitializeAsync` on a live driver with *no*
|
||||
teardown, so a driver that does `_devices[host] = new DeviceState(...)` (or `Session = ...`, `Plc = ...`) orphans
|
||||
the only way in: the `IDriver` contract allows `InitializeAsync` on a live driver with *no* teardown, and a retry
|
||||
tick can overlap a still-running attempt, so a driver that does `_devices[host] = new DeviceState(...)` (or `Session = ...`, `Plc = ...`) orphans
|
||||
the previous state's probe / recycle / fixed-tree loops and its client. FOCAS, AbLegacy, AbCip, TwinCAT, S7 and
|
||||
OpcUaClient now cancel, bounded-join (5 s) and dispose that state at the top of `InitializeAsync`
|
||||
(`ReleaseDeviceStatesAsync` / `ReleaseConnectionAsync` / `ReleaseSessionAsync`), alongside Modbus and MQTT.
|
||||
@@ -598,8 +657,9 @@ Implementation:
|
||||
`GatewayAlarmHistorianWriter` (the gateway `SendEvent` path; see
|
||||
[AlarmHistorian.md](AlarmHistorian.md)).
|
||||
|
||||
The HistorianGateway is the sole historian backend; its config keys and
|
||||
deployment prerequisites are in [Historian.md](Historian.md).
|
||||
The historian backend is pluggable (`ServerHistorian:Backend` = `Gateway`, the default, or `SqlServer`);
|
||||
the HistorianGateway is the Gateway backend. Config keys and deployment prerequisites for both are in
|
||||
[Historian.md](Historian.md).
|
||||
|
||||
---
|
||||
|
||||
|
||||
+1812
-61
File diff suppressed because it is too large
Load Diff
@@ -16,9 +16,9 @@ changes the served address space — the **deploy artifact** — and one advisor
|
||||
check (`FetchAndCache`) — see `docs/Configuration.md` §`ConfigSource`.
|
||||
3. **Apply.** `DriverHostActor` (`src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Drivers/`) opens the artifact and
|
||||
reconciles in two halves:
|
||||
- **Drivers** — `DriverSpawnPlanner.Compute` yields `ToStop` / `ToApplyDelta` / `ToSpawn`. A changed
|
||||
driver config reaches the live driver as `IDriver.ReinitializeAsync` on its `DriverInstanceActor`;
|
||||
a removed driver is stopped. An unreadable or empty blob is "no answer": the apply fails and the
|
||||
- **Drivers** — `DriverSpawnPlanner.Compute` yields `ToStop` / `ToSpawn`. A changed
|
||||
driver config (DriverConfig, DriverType or ResilienceConfig) is a stop + respawn (#516) — there is no
|
||||
in-place path (#665 deleted the unreachable `ApplyDelta` seam); a removed driver is stopped. An unreadable or empty blob is "no answer": the apply fails and the
|
||||
running children are left alone (never an empty configuration).
|
||||
- **Address space** — `AddressSpaceComposer` builds an `AddressSpaceComposition` from the artifact;
|
||||
`AddressSpacePlan` diffs it against the last applied composition; `AddressSpaceChangeClassifier`
|
||||
|
||||
+10
-1
@@ -546,7 +546,16 @@ Under warm/hot redundancy both cluster nodes run `ScriptedAlarmHostActor` and ev
|
||||
node. ⚠️ **The peer it stands down for must record the same refs** — a Primary in its own per-cluster
|
||||
mesh — not merely hold the same rows; a node with no cluster-scoped role never stands down, because a
|
||||
fleet-wide election's Primary usually runs a different cluster's configuration and deferring to it would
|
||||
end this cluster's historization silently. See [Historian.md](Historian.md).
|
||||
end this cluster's historization silently. **Crash failover (Gitea #672):** a SIGKILLed Primary stays an
|
||||
unreachable-but-not-downed member until auto-down (10 s failure detector + 15 s), and the redundancy
|
||||
singleton usually died with it, so the standby's cached snapshot kept naming it Primary and nobody
|
||||
captured for ~30 s. `DriverHostActor` now follows `ClusterEvent.ReachabilityEvent` and excludes an
|
||||
**unreachable** peer from the historization peer-is-Primary question only (re-deriving the capture verdict
|
||||
from the cached snapshot), shrinking the gap to the failure-detector window; gated by
|
||||
`ContinuousHistorization:CaptureWhilePrimaryUnreachable` (default `true`). The alarm drain and the Primary
|
||||
data-plane verdict are **unchanged** — they still wait for a snapshot. The cost is duplicate capture while a
|
||||
live Primary is merely unreachable, and from detection onward under a real partition. See
|
||||
[Historian.md](Historian.md) §"A crashed Primary: capture gap (#672)".
|
||||
|
||||
Net effect: each alarm transition appears **once** on `/alerts` and would historize once, not once per node.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ A production OtOpcUa deployment runs **one binary per node**. The historian back
|
||||
| Process | Project | Runtime | Platform | Responsibility |
|
||||
|---|---|---|---|---|
|
||||
| **OtOpcUa Host** | `src/Server/ZB.MOM.WW.OtOpcUa.Host` | .NET 10 | AnyCPU | Single fused binary. The resolved [`EffectiveRoles`](#role-gating) decides what to mount: `admin` (Blazor + auth + control-plane singletons), `driver` (OPC UA endpoint + per-driver actors), or both. |
|
||||
| **ZB.MOM.WW.HistorianGateway** *(external — separate deployment)* | not in this repo | .NET 10 | — | The sole historian backend. OtOpcUa talks gRPC to it (via the `ZB.MOM.WW.HistorianGateway.Client` package) for HistoryRead, alarm `SendEvent`, and continuous `WriteLiveValues`. Must run `RuntimeDb:Enabled=true` + `RuntimeDb:EventReadsEnabled=true`; the API key must carry `historian:read` + `historian:write` + `historian:tags:write`. |
|
||||
| **ZB.MOM.WW.HistorianGateway** *(external — separate deployment)* | not in this repo | .NET 10 | — | The default historian backend (`ServerHistorian:Backend=Gateway`; `SqlServer` needs no sidecar). OtOpcUa talks gRPC to it (via the `ZB.MOM.WW.HistorianGateway.Client` package) for HistoryRead, alarm `SendEvent`, and continuous `WriteLiveValues`. Must run `RuntimeDb:Enabled=true` + `RuntimeDb:EventReadsEnabled=true`; the API key must carry `historian:read` + `historian:write` + `historian:tags:write`. |
|
||||
|
||||
Galaxy access still uses the separately-installed **mxaccessgw** sidecar (see `docs/v2/Galaxy.ParityRig.md`); the gateway owns the MXAccess COM bitness constraint (its worker is x86 net48). Nothing in the OtOpcUa repo carries that constraint anymore. (The bespoke Wonderware historian sidecar this deployment used to ship was retired — see [drivers/Historian.Wonderware.md](drivers/Historian.Wonderware.md).)
|
||||
|
||||
@@ -32,13 +32,13 @@ Single-node dev: `OTOPCUA_ROLES=admin,driver` (or the `Cluster:Roles` equivalent
|
||||
|
||||
### Per-role configuration overlays
|
||||
|
||||
`Program.cs` builds a role suffix by joining the resolved `EffectiveRoles.Gossiped` roles **alphabetically** with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role (`admin`/`driver`/`dev`) is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename for those is stable (`appsettings.driver.json`); if a `cluster-{ClusterId}` role is also configured, its suffix carries into the filename **verbatim** (e.g. `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.cluster-SITE-A-driver.json`), since `Gossiped` preserves that spelling by design (see [Configuration.md § `Roles`](Configuration.md#appsettings-sections)). Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
`Program.cs` builds a role suffix (`RoleOverlay.Suffix`) by joining the resolved `EffectiveRoles.Gossiped` **plain** roles (`admin`/`driver`/`dev`) alphabetically with `-` and loads `appsettings.{roleSuffix}.json` as an optional overlay on top of base `appsettings.json`. A plain role is always lower-case in `Gossiped` regardless of how the operator spelled it, so the overlay filename is stable (`appsettings.driver.json`). **`cluster-{ClusterId}` roles are NOT part of the filename** (#678.6): they are mesh-membership tags, so `Cluster:Roles: ["driver", "cluster-SITE-A"]` loads `appsettings.driver.json` and `["admin", "driver", "cluster-MAIN"]` loads `appsettings.admin-driver.json`. (Before #678.6 the cluster role was folded into the name, producing e.g. `appsettings.admin-cluster-MAIN-driver.json`, which never exists — so on every node with a cluster role no shipped overlay ever loaded.) Three overlays ship in `src/Server/ZB.MOM.WW.OtOpcUa.Host/`:
|
||||
|
||||
- `appsettings.admin.json` — admin-only nodes
|
||||
- `appsettings.driver.json` — driver-only nodes
|
||||
- `appsettings.admin-driver.json` — fused single-node dev / small deployments
|
||||
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false`. Configuration loading order (lowest to highest precedence) is:
|
||||
All three carry Serilog log-level overrides + `Security:Ldap:DevStubMode = false` + `Security:Ldap:Transport = Ldaps` (audit table: [Configuration.md § Role overlays](Configuration.md#role-overlay-audit)). Configuration loading order (lowest to highest precedence) is:
|
||||
|
||||
`appsettings.json` < `appsettings.{Environment}.json` < `appsettings.{role}.json` < environment variables < command-line args
|
||||
|
||||
|
||||
+24
-11
@@ -165,8 +165,8 @@ The driver has two interchangeable wire transports, selected by one `DriverConfi
|
||||
the connection size. A read whose size cannot be bounded (structure / unknown authored type) is sent
|
||||
alone; writes and status-only replies count their 4-byte header. Pinned by
|
||||
`RequestSchedulerTests.Mixed_scalar_string_and_array_replies_are_packed_within_the_reply_budget`.
|
||||
⚠️ RESIDUAL: mixed STRING/array packing has not been exercised against a real controller (the real-press
|
||||
gate predates the budget); the budget is unit-tested against the scheduler and fake channel only.
|
||||
Exercised on a real CompactLogix (Z-35162, hw4, 2026-10-09): a STRING, an array element, a DINT[32] and two
|
||||
scalars rode one Multiple Service Packet in 5/5 attempts, values equal to single reads.
|
||||
- **Fragmented-read integrity (#567):** `Read Tag Fragmented` replies are compared against the first reply;
|
||||
a changed type code *or* structure handle mid-read fails the read with `BadReply` rather than stitching
|
||||
two types into one value.
|
||||
@@ -179,9 +179,16 @@ The driver has two interchangeable wire transports, selected by one `DriverConfi
|
||||
and the next request reopens through the throttle. The request that got the reply fails
|
||||
`BadCommunicationError` (transport `BadConnection`); a keep-alive that gets it recycles the connection
|
||||
before any request fails. Every other status — tag-level errors, other `0x01` extended codes — is left to
|
||||
the request. ⚠️ RESIDUAL: which of the two codes a real Logix controller returns for a forgotten
|
||||
connection (and whether it answers at all rather than dropping TCP) is unobserved; the check is correct
|
||||
whichever it does — a silent target still faults the session through the exchange timeout.
|
||||
the request. **Observed on a real CompactLogix (Z-35162, hw5, 2026-10-09): neither code.** A connection left
|
||||
idle 60 s, past its 16 s timeout (keep-alive disabled for the test), was closed at the TCP level somewhere in
|
||||
that 16–60 s window — the exact moment, and whether the controller sent RST or FIN, were not captured (no
|
||||
packet capture; EPIPE on the first write only says the close was complete by then); the next request
|
||||
failed on its write in 9 ms (`Broken pipe` → `BadConnection` → `BadCommunicationError`, no CIP or
|
||||
encapsulation status) and the request after it reopened with a fresh Forward Open. That path needs no CIP
|
||||
check — a faulted session always reopens on the next send — and is pinned offline by
|
||||
`CipConnectionTests.A_target_that_resets_an_idle_session_fails_the_next_request_fast_and_the_one_after_reopens`.
|
||||
The 0x0107/0x0203 recycle stays for targets that keep TCP up. In production the keep-alive (every 8 s) stops
|
||||
a held connection idling out, so this shape follows a controller restart / re-download or a network gap.
|
||||
- `ConnectionSize` / `Timeout` / `AllowPacking` are honoured per pooled connection, and the **first**
|
||||
driver/device to open a given device key wins — including a Test Connect probe on a fused `admin,driver`
|
||||
node (its connection carries the probe timeout). A later opener with different options is ignored and
|
||||
@@ -241,16 +248,22 @@ The driver has two interchangeable wire transports, selected by one `DriverConfi
|
||||
and templates on Z-35162; a user-approved one-off on 2026-10-09 added **scalar native writes on a real CompactLogix**
|
||||
(DINT `test2`, BOOL `Program:UN01_Press.xxtest`, REAL `Program:ES03_EM03_Zone3._ff_test`: write, read back, restore, all
|
||||
Good), and the #611 gate added **native UDT member writes** (DINT, BIT and REAL members of `_EmptyTemperatureSegment`).
|
||||
Still ab_server-only: array and STRING writes, libplctag member writes, and fragmented writes. Mixed STRING/array/scalar
|
||||
hw5 (2026-10-09) added a **native STRING write** (`Program:ES03_TemperatureControl._StrTmp`, through the driver:
|
||||
write, read back, restored byte-exact). A user-approved **native array-element write** followed: `Program:EM01_PowerUnit._SEQ_EP[31]`
|
||||
0→4242→0 (the array's only alias targets `[1]`, no routine indexes or block-copies it; all 32 elements snapshotted
|
||||
twice before, after the write and after the restore — only `[31]` changed). Still ab_server-only: whole-array
|
||||
writes (the driver has none), libplctag member writes, and fragmented writes. #566 is answered (the controller drops TCP; see "A target-closed CIP connection"). Mixed STRING/array/scalar
|
||||
Multiple Service Packet reads (#582) are proven on the press (one 5-read MSP, values equal to single reads). Results:
|
||||
[native-ab-transport-gate-results](../plans/2026-10-07-native-ab-transport-gate-results.md).
|
||||
- **libplctag cannot be deleted yet.** Scalar native writes are proven on a CompactLogix (2026-10-09); what remains is #567
|
||||
(fragmented wide-struct writes), array/STRING writes on hardware, and AbLegacy native (simulator-only, no PCCC hardware).
|
||||
- **libplctag cannot be deleted yet.** Scalar, UDT-member and STRING native writes are proven on a CompactLogix
|
||||
(2026-10-09); what remains is #567 (fragmented wide-struct writes), whole-array writes on hardware, and AbLegacy
|
||||
native beyond the MicroLogix 1400 evidence.
|
||||
- UDT **member writes** (#611) are proven on a real CompactLogix on the **native** transport only (2026-10-09); the
|
||||
libplctag member-write path is `ab_server`-only. See [Member writes](#member-writes).
|
||||
- ⚠️ RESIDUAL (#567): wide-struct **fragmented writes** (8-byte alignment) are unit-tested only. Production
|
||||
presses are read-only and there is no bench controller, so this cannot be proven; it stays open until a
|
||||
bench controller exists.
|
||||
- ⚠️ RESIDUAL (#567): wide-struct **fragmented writes** (8-byte alignment) are unit-tested only. A write fragments only
|
||||
above `MaxRequestBytes` (4000 B on a Large Forward Open), so it needs a ~4 KB value; Z-35162 has no safe one (a DINT[32]
|
||||
or a STRING is a single Write Tag, and AbCip has no whole-array write anyway). It stays open until a bench controller
|
||||
or an approved large scratch tag exists.
|
||||
- Native browse (#572, resolved in code): listing paging no longer assumes ascending instance order, pages
|
||||
are paced, and a browse's template reads share one lease. Observed on Z-35162 (2026-10-09, 41
|
||||
pages / 3162 tags): every page strictly ascending, so out-of-order listing has never been seen on a real controller — the
|
||||
|
||||
@@ -100,7 +100,7 @@ physical inputs and may reject the write.
|
||||
re-parses carry identical numbers. This is not a double conversion: libplctag's `pccc.c`
|
||||
(`parse_pccc_elem_num` / `parse_pccc_bit_num`) parses decimal numerals only (a FIXME there says octal is
|
||||
unsupported), so the driver must convert. Pinned by `Plc5_io_indices_are_octal` (`I:017/17` -> `I:15/15`).
|
||||
The connectivity-probe address is normalised the same way.
|
||||
The connectivity-probe address is normalised the same way — by both the driver's probe loop and the Test Connect probe, through one helper (`AbLegacyProbeAddress.Normalise`, #660); an unparseable Test Connect probe address fails cleanly ("not a valid AbLegacy address") instead of reaching the transport. Normalisation yields the canonical name; the libplctag `S:0` → `S2:0` translation (#659) happens downstream in the runtime, so the two compose.
|
||||
- ⚠️ **Behaviour change / migration.** Before this change a PLC-5 `I:`/`O:` address was passed to libplctag as
|
||||
written, i.e. read as **decimal**. An address copied from RSLogix in octal (e.g. `I:017`) therefore read the
|
||||
wrong word, and an authored decimal such as `I:1/12` now means bit 10 (octal 12). A previously "working"
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
|
||||
> **This backend has been retired.** The bespoke Wonderware TCP/ArchestrA historian sidecar
|
||||
> (`OtOpcUaWonderwareHistorian`) and its `Driver.Historian.Wonderware*` projects — plus the vestigial
|
||||
> `Historian.Wonderware` driver type — were removed. **HistorianGateway is now the sole historian
|
||||
> backend** for OtOpcUa (read, alarm-write, and continuous historization).
|
||||
> `Historian.Wonderware` driver type — were removed. **HistorianGateway is now the default historian
|
||||
> backend** for OtOpcUa (read, alarm-write, and continuous historization); a plain SQL Server backend is the
|
||||
> alternative (`ServerHistorian:Backend=SqlServer`, see [Historian.md](../Historian.md)).
|
||||
|
||||
## What replaced it
|
||||
|
||||
|
||||
@@ -415,7 +415,7 @@ down rather than swallowing it. A SUBACK failure degrades just the affected RawP
|
||||
There are **two** kinds of reconnect and only one of them fires that callback. A drop the connection's
|
||||
own supervisor recovers keeps the same `MqttConnection`, so `Reconnected` → `OnReconnectedAsync`
|
||||
re-establishes every live filter. A **session rebuild** — `ShutdownAsync` + `InitializeAsync` (the #527
|
||||
comms-loss watchdog's retry path, and an `ApplyDelta` that changes the endpoint or credentials) — builds
|
||||
comms-loss watchdog's retry path, and a `ReinitializeAsync` that changes the endpoint or credentials) — builds
|
||||
a *new* connection and re-attaches the *same* `MqttSubscriptionManager`, and the re-subscribe then comes
|
||||
from the host (`DriverInstanceActor.ResubscribeDesired`) rather than from the callback. `AttachTo`
|
||||
therefore **forgets the filters the previous session had established**: the dedupe set is a claim about
|
||||
|
||||
+148
-15
@@ -45,22 +45,139 @@ out-of-process isolation.
|
||||
```
|
||||
|
||||
`EndpointUrls` (a list) takes precedence over the single-URL `EndpointUrl` and
|
||||
provides ordered **failover** — the driver tries each candidate in turn at init
|
||||
and on session drop, and the first to connect wins (e.g. a hot-standby pair on
|
||||
provides ordered **failover** — the driver tries each candidate in turn at every
|
||||
`InitializeAsync` (including the comms-loss watchdog's re-initialise), and the first to connect wins (e.g. a hot-standby pair on
|
||||
4840 / 4841). See
|
||||
`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.Contracts/OpcUaClientDriverOptions.cs`
|
||||
for every field (security policy/mode, auth type, session timeout, keep-alive,
|
||||
reconnect period, browse root, node/depth caps).
|
||||
browse root, node/depth caps). `ReconnectPeriod` was **deleted** (#676 — unused since #670 removed the SDK reconnect
|
||||
handler, and the AdminUI no longer shows it); stored configs that still carry `reconnectPeriod` keep parsing in the
|
||||
factory, the Test-Connect probe and the browse picker (unknown keys are skipped), and the form drops it on the next save.
|
||||
|
||||
### Endpoint address: the configured host and port win (#676)
|
||||
|
||||
GetEndpoints returns each endpoint with the URL the **server** believes it lives at. Behind NAT, a port map, a
|
||||
container port publish or a proxy that address is unreachable from the client — or reachable but bypassing the proxy.
|
||||
The driver therefore rewrites the selected endpoint's **host and port** to the configured URL's whenever the schemes
|
||||
match, keeping the advertised path (the SDK's own `CoreClientUtils.SelectEndpoint` rule, which the driver's
|
||||
policy-aware endpoint selection used to bypass). A configured URL with **no port** keeps the advertised port (copying
|
||||
the port-less `-1` would have sent the session to 4840). A different scheme is left alone; an unparseable advertised
|
||||
URL falls back to the configured one. `OpcUaClientDriver.RewriteEndpointUrl`.
|
||||
|
||||
### Session lifetime
|
||||
|
||||
A single `Session` per driver instance; subscriptions multiplex onto it. The
|
||||
SDK reconnect handler takes the session down and brings it back on remote-server
|
||||
restart, re-sending subscriptions on reconnect so monitored-item handles don't
|
||||
dangle. Stored NodeIds embed the server-stable namespace **URI** (not the
|
||||
A single `Session` per driver instance; subscriptions multiplex onto it. Stored NodeIds embed the server-stable namespace **URI** (not the
|
||||
session-relative `ns=N` index) so a remote namespace-table reorder across a
|
||||
restart doesn't silently re-point references at the wrong namespace.
|
||||
|
||||
### Upstream outage and recovery (Gitea #670)
|
||||
|
||||
The session keep-alive (`KeepAliveInterval`, default 5 s) is the liveness signal:
|
||||
|
||||
- **Good keep-alive / any data or event notification** advances `DriverHealth.LastSuccessfulRead` — a
|
||||
subscription-only driver never calls `ReadAsync`, so without this its read stamp froze at connect time.
|
||||
- **Bad keep-alive** ⇒ health `Degraded` (`LastError = "keep-alive: <status>"`, read stamp preserved), host status
|
||||
`Stopped`, and — **once per outage**, not per keep-alive — every subscribed ref is published `BadNoCommunication`
|
||||
with its last value retained and **source + server timestamp = the detection time** (live-gate F1: stamped with the
|
||||
last Good's source time it collided with that row's `(TagId, SourceTimeUtc)` history key and was never stored), plus a quality-only `BadNoCommunication` annotation (#626 pattern, no alarm-state
|
||||
change, no `/alerts` row) on every subscribed alarm condition.
|
||||
- **The session survives** (a network blip; the keep-alive turns Good again) ⇒ recovered in place: health `Healthy`,
|
||||
the refs still Bad are re-read once and published Good (a server whose subscription survived does not resend an
|
||||
unchanged value, and a `(value, status)` dedup downstream lets the identical value through), alarm quality goes
|
||||
back Good and a `ConditionRefresh` re-reports anything missed. A notification that lands first wins over the read.
|
||||
For a bounded **replay window** after such a recovery — max(30 s, 2 x the longest subscription keep-alive window) —
|
||||
each ref is guarded by a high-water mark — its last Good upstream source time, raised by the recovery read and
|
||||
advanced by every notification accepted in the window — and a notification whose source time is at or BELOW it
|
||||
(equal included: OPC UA moves the source timestamp on every value or status change, so an equal one is a replay)
|
||||
is dropped: the SDK was seen delivering a missed one 4–6 s late, which regressed the node (live-gate F2). The
|
||||
guard holds for the WHOLE window — fresh notifications arrive ~1 s after recovery, before the late replay, so a
|
||||
ref is never disarmed early (the F2 re-gate failed on exactly that). A notification with no source time passes
|
||||
without moving the mark. Outside that window every notification is
|
||||
accepted whatever its timestamp, so an upstream clock that steps backwards (NTP, a PLC clock reset) never
|
||||
freezes a node. The outage Bad never moves the comparison; a re-initialise resets it.
|
||||
⚠️ **By design, inside the window a notification whose source time EQUALS the last accepted one is dropped** — so a
|
||||
status-only change that keeps the old source timestamp (not what OPC UA servers are supposed to do) is lost during
|
||||
the window only (the trailing re-read below repairs it). When the window closes — or a second recovery re-arms it —
|
||||
the driver logs ONE line, `post-recovery replay window closed — dropped N late notifications across M refs after
|
||||
recovery`, at **Information when N > 0** and Debug otherwise, because the per-drop line is Debug and the
|
||||
`stale_notification_dropped` counter is OTel-only, which made the guard unobservable on the docker-dev rig (#676).
|
||||
A re-arm starts the count afresh.
|
||||
- **Trailing re-read after an in-place recovery (#676).** Found by the TCP-stall wire test: a change the server
|
||||
published just after the stall ended rode a Publish response that had been in flight across the stall; the SDK
|
||||
timed that request out (`BadTimeout`), discarded the response and never republished the message (the following
|
||||
keep-alives moved the sequence number past it), so the node stayed **Good on the pre-change value indefinitely**.
|
||||
`Timeout + KeepAliveInterval` after the recovery — when every request in flight across the stall has completed or
|
||||
timed out — every subscribed ref is read once more and published when its **(value, status) differs** from the last
|
||||
value delivered for that ref **and is not older** than it. Any status counts: a Good→Bad/Uncertain change lost in the
|
||||
pipeline is as stale as a lost value. "Older" compares source timestamps only when both carry one; a read with no
|
||||
source timestamp (a server that does not send them) is the server's current state and counts as newer. An **equal**
|
||||
source timestamp with different content IS published — it is the server's current state, not a replay, and it repairs
|
||||
the equal-timestamp status-only change the replay guard drops by design. An identical (value, status) is never
|
||||
republished. A ref that received ANY delivery while the read was in flight is skipped (a per-ref delivery counter is
|
||||
captured before the read is issued): without source timestamps the read's sample cannot be ordered against that
|
||||
notification, and publishing it would regress the node. An Information line names how many refs it had to repair. The trailing read and the replay-guard disarm
|
||||
timer run on the driver clock and are cancelled when the session is released (re-initialise / shutdown / dispose).
|
||||
- **No older value is ever delivered after a newer one.** The paths that publish upstream data — live notifications,
|
||||
the recovery read, the trailing read and the comms-loss Bad fan — make their "may I publish this?" decision and the
|
||||
publish inside one driver-wide critical section. A single lock rather than one per ref: it also serialises delivery
|
||||
across subscriptions (the SDK dispatches each subscription's messages on its own worker), which is cheap because each
|
||||
section is a dictionary update plus a subscriber `Tell` and nothing awaits under it. Subscribers run inside it — the
|
||||
host's handler only `Tell`s an actor — so a subscriber must never block on, or call back into, the driver.
|
||||
- **The session does not come back** (an upstream restart: the server forgot the session) ⇒ the driver stays
|
||||
`Degraded` with its read stamp frozen, which is the #527 comms-loss watchdog's trip condition; `DriverInstanceActor`
|
||||
runs `ShutdownAsync` + `InitializeAsync` (retried every 10 s while the server is down) and re-subscribes data +
|
||||
alarms; the new subscription's initial values come back Good. Alarm last-told state survives the re-initialise
|
||||
(#619), so the post-subscribe `ConditionRefresh` raises no un-ack flicker; the host's reconnect annotation restores
|
||||
condition quality.
|
||||
- **Fast recovery for a DEFINITIVELY lost session (#676).** The driver implements `IDriverConnectionLossSource` and
|
||||
raises `ConnectionLost` — **once per session**, re-armed by a re-initialise — when the loss is certain, so the host
|
||||
re-initialises at once instead of after the watchdog's 90 s. Definitive means: a keep-alive status of
|
||||
`BadSessionIdInvalid` / `BadSessionClosed` / `BadSessionNotActivated` (the server forgot, closed or never activated
|
||||
the session) or `BadServerHalted`, or a subscription the **server** reported gone (below). `BadServerHalted` is on the list
|
||||
because it was **measured**: on the restart wire test a gracefully stopped server reports `BadServerHalted` (and the
|
||||
keep-alives alternate it with `BadNoCommunication`), and the SDK never re-opens a channel the server halted — 15 s
|
||||
after the server was back, the same session still failed every request with `BadServerHalted`; `BadSessionIdInvalid`
|
||||
never arrived.
|
||||
**A CLOSED connection is definitive too, once sustained (live gate A1).** Against a real opc-plc on docker-dev, none of
|
||||
`docker restart`, a graceful SIGINT stop (opc-plc itself logged `BadServerHalted`) or `docker kill` delivered a
|
||||
session status to the client: the keep-alives reported `BadSecureChannelClosed` (0x80860000) once, then
|
||||
`BadConnectionClosed` (0x80AE0000) interleaved with `BadNoCommunication` for the whole outage, so the first cut of the
|
||||
fast path never fired and recovery took 91–127 s. Reproduced in-process with `TcpStallProxy.HardClose` (every
|
||||
connection reset, new ones refused, the server AND its session still alive behind it): the same sequence, and the SDK
|
||||
1.5.378 session **never opens a new connection by itself** — 48 s after the proxy accepted connections again, not one
|
||||
attempt. So a closed channel can never recover in place; on the **second** closed-channel keep-alive failure in one
|
||||
outage (`IsClosedChannel`; a good keep-alive resets the count, so a single transient does not count) the driver raises
|
||||
`ConnectionLost` — about one keep-alive interval after the connection dropped. Everything else —
|
||||
`BadNoCommunication`, `BadNotConnected`, `BadRequestTimeout`/`BadTimeout` (what a stalled or paused link produces) —
|
||||
is **ambiguous** (the link may resume and keep its session, see the stall test) and stays with the watchdog.
|
||||
Consequence: a restarted, stopped or killed upstream is re-initialised within roughly one keep-alive plus one
|
||||
reconnect interval instead of ~100 s; a pause (`docker pause`, a congested link) still recovers in place. ⚠️ **Known gap:** the host acts on the signal only while the driver is
|
||||
`Connected`, and the driver raises it once per session — so a definitive loss raised while the host is not
|
||||
Connected (e.g. `BadServerHalted` on the fresh session before the host has processed the successful initialise) is
|
||||
swallowed and that session falls back to the watchdog. Re-raising on later keep-alives would close it but was left out
|
||||
to keep the once-per-session contract simple; the cost is the pre-#676 recovery time, in a narrow window.
|
||||
- **The subscription dies but the session lives** (an outage longer than the subscription lifetime —
|
||||
`LifetimeCount` x publishing interval, often well under the session timeout — or a server that drops it) ⇒ keep-alives
|
||||
stay Good, so they cannot be the signal. A `PublishError` of `BadNoSubscription` (while subscriptions are tracked) or
|
||||
`BadSubscriptionIdInvalid` naming a tracked subscription, or a tracked subscription whose SDK `PublishingStopped`
|
||||
persists for one more subscription keep-alive window + 5 s across good keep-alives, declares it lost: `Degraded`
|
||||
(`"subscription lost: …"`), the outage published once, and — sticky until the session is released — **nothing**
|
||||
advances `LastSuccessfulRead` any more (not good keep-alives, not a sibling subscription's notifications), so the
|
||||
watchdog's re-initialise recreates it. Both detectors are wire-tested independently. A subscription the **server**
|
||||
reported gone (`BadNoSubscription` / `BadSubscriptionIdInvalid`) never comes back on its session, so that also raises
|
||||
`ConnectionLost` (#676) and the re-initialise happens at once; the `PublishingStopped` **heuristic** is an inference
|
||||
from silence and keeps going through the watchdog.
|
||||
- An in-place recovery is bound to the session whose keep-alive recovered: one overtaken by a re-initialise neither
|
||||
publishes, nor degrades, nor re-arms the outage latch of the new session.
|
||||
|
||||
There is **no in-driver reconnect** any more. The SDK's `SessionReconnectHandler` (removed) retried forever in SDK
|
||||
1.5.378 (its "exhausted" path was unreachable), recreated sessions by cloning subscriptions the driver never
|
||||
re-tracked, and kept health `Healthy` throughout — so on docker-dev a stopped opc-plc left nodes Good on their last
|
||||
value indefinitely and they never recovered. A teardown against a lost session skips the wire deletes and `CloseSession`
|
||||
(bounded to 5 s otherwise). SDK warnings (keep-alive / publish / channel errors) now reach the driver's own logger,
|
||||
rate-limited to 20 per minute with a suppressed-count note; the per-driver SDK logger factory is disposed with the
|
||||
driver.
|
||||
|
||||
### Namespace assignment (retired, Gitea #648)
|
||||
|
||||
Earlier versions required a per-instance `TargetNamespaceKind` (`Equipment` +
|
||||
@@ -79,7 +196,7 @@ initialise (the AdminUI form drops them on the next save). Pinned by
|
||||
|
||||
## Capability surface
|
||||
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource`
|
||||
`OpcUaClientDriver : IDriver, ITagDiscovery, IReadable, IWritable, ISubscribable, IHostConnectivityProbe, IAlarmSource, IDriverConnectionLossSource`
|
||||
(`src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/OpcUaClientDriver.cs:31`).
|
||||
|
||||
| Capability | Path | Notes |
|
||||
@@ -101,14 +218,17 @@ initialise (the AdminUI form drops them on the next save). Pinned by
|
||||
> `Raise`/`Acknowledge`/`Clear`/`Retrigger` kinds the host projector consumes (without a kind no upstream alarm could ever
|
||||
> show active); the kinds are emitted from the state **last told to the host** per RawPath, so an identical re-delivery
|
||||
> emits nothing (no re-raise, which would un-ack). An inactive → active+acked move is ONE `ActiveAcknowledged` (#656; it
|
||||
> used to be `Raise` + `Acknowledge`, a client-visible unacked intermediate). The first event per condition after a
|
||||
> start or deploy respawn is flagged `IsRestatement`, so the host writes no duplicate "Activated" row when its
|
||||
> condition already shows that state (see `docs/AlarmTracking.md`). Condition **branch** events (non-null `BranchId`) are ignored.
|
||||
> **`ConditionRefresh`** is called after the event item is created, after every in-place reconnect, and on a
|
||||
> used to be `Raise` + `Acknowledge`, a client-visible unacked intermediate); a move into inactive+unacked is ONE
|
||||
> `InactiveUnacknowledged` (#657; it used to be `Raise`/`Retrigger` + `Clear`, an Active flicker). The first event per
|
||||
> condition after a start or deploy respawn is flagged `IsRestatement` and states the upstream condition so it lands
|
||||
> from whatever the host retained: an inactive+acked one goes out as `Clear` + `Acknowledge` (#657; it used to be an
|
||||
> `Unspecified` the host wrote as "Activated", and a retained-Active host condition stayed Active). So the host writes no
|
||||
> duplicate row when its condition already shows that state (see `docs/AlarmTracking.md`). Condition **branch** events (non-null `BranchId`) are ignored.
|
||||
> **`ConditionRefresh`** is called after the event item is created, after a keep-alive outage the session survived, and on a
|
||||
> `RefreshRequired` event, so a condition already active upstream is reported at subscribe; `RefreshStart`/`RefreshEnd`
|
||||
> markers are dropped. A server without it (`BadNotSupported`/`BadMethodInvalid`/…) is logged once and tolerated.
|
||||
> Last-told state survives a re-initialise (the host re-subscribes the same set over nodes it did not rebuild) and is
|
||||
> forgotten when the subscribed set changes (the host rebuilt, recreating the nodes inactive). Index keys are server-stable (`nsu=`), so a namespace reorder across a reconnect keeps
|
||||
> forgotten when the subscribed set changes (the host rebuilt, recreating the nodes inactive). Index keys are server-stable (`nsu=`), so a namespace reorder across a re-initialise keeps
|
||||
> matching. docker-dev seeds an opc-plc driver (`MAIN-opcua`); the alarm tag is
|
||||
> created only when `@OpcPlcAlarmSourceNodeId` is set in `docker-dev/seed/seed-clusters.sql`.
|
||||
|
||||
@@ -189,8 +309,10 @@ every `InitializeAsync`, including comms-loss reconnects). The discovery channel
|
||||
Session/publish events are published on the shared OTel meter as the counter
|
||||
`otopcua.driver.opcuaclient.events`, tagged `driver_instance` (the driver instance id) and `event` =
|
||||
`publish_response`, `notification` (items), `data_change`, `bad_status_value`, `publish_error`,
|
||||
`keepalive_miss`, `reconnect_attempt`, `reconnect_success`, `reconnect_exhausted`. A climbing `keepalive_miss` /
|
||||
`reconnect_attempt` rate is the signature of a flapping upstream. They are not part of `DriverHealth` or the
|
||||
`keepalive_miss`, `keepalive_recovered` (a bad keep-alive followed by a good one on the same session),
|
||||
`stale_notification_dropped` (a late replay dropped inside the post-recovery window, see above). A climbing
|
||||
`keepalive_miss` / `keepalive_recovered` rate is the signature of a flapping upstream (`reconnect_*` were removed with
|
||||
the reconnect handler, #670). They are not part of `DriverHealth` or the
|
||||
`/hosts` panel.
|
||||
|
||||
## Testing
|
||||
@@ -201,6 +323,17 @@ Session/publish events are published on the shared OTel meter as the counter
|
||||
- **Integration fixture** — exercises the driver against a reference OPC UA
|
||||
server (opc-plc) on the shared docker host; see
|
||||
[OpcUaClient-Test-Fixture.md](OpcUaClient-Test-Fixture.md) for the coverage map.
|
||||
- **Wire tests against the in-process server** — `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/`
|
||||
`OpcUaClientDriver*WireTests` (no Docker). `OpcUaClientDriverStallWireTests` (#676) covers the in-place blip that
|
||||
used to be live-gate-only: the driver connects through `TcpStallProxy`, an in-test TCP forwarder whose pause holds
|
||||
every byte without closing anything (which only works because of the endpoint rewrite above); it asserts one Bad,
|
||||
a frozen read stamp, recovery on the SAME session, the identical value republished Good, a change published right
|
||||
after recovery still arriving, and no source-time regression once the SDK's late backlog flush has arrived. Its
|
||||
second test uses `HardClose` (reset every connection, refuse new ones) — the shape of a real process stop — and
|
||||
asserts the live-gate status sequence, that the SDK never reconnects on its own once the proxy accepts again,
|
||||
`ConnectionLost` exactly once, and the re-initialise bringing the identical value back Good.
|
||||
`OpcUaClientDriverReconnectWireTests` covers the server restart (incl. the `BadServerHalted` → `ConnectionLost` signal)
|
||||
and the subscription-lost case.
|
||||
|
||||
## Further reading
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ OtOpcUa is a multi-driver OPC UA server. The Core (`ZB.MOM.WW.OtOpcUa.Core` + `C
|
||||
- `IAlarmSource` — driver-emitted OPC UA A&C events
|
||||
- `IRediscoverable` — driver-initiated address-space rebuild notifications
|
||||
- `ISkippedTagReporter` — authored tags the driver could not build into its poll table (see [Skipped tags](#skipped-tags) below)
|
||||
- `IHistorianDataSource` — server-side historian read backend registration (the HistorianGateway backend) (drivers carry no history capability; Gitea #655)
|
||||
- `IHistorianDataSource` — server-side historian read backend registration (the selected `ServerHistorian:Backend` — HistorianGateway or SQL Server) (drivers carry no history capability; Gitea #655)
|
||||
|
||||
Each driver opts into only the capabilities it supports. Every async capability call at the Server dispatch layer goes through `CapabilityInvoker` (`Core/Resilience/CapabilityInvoker.cs`), which wraps it in a Polly pipeline keyed on `(DriverInstanceId, HostName, DriverCapability)`. The `OTOPCUA0001` analyzer enforces the wrap at build time. Drivers themselves never depend on Polly; they just implement the capability interface and let the Core wrap it.
|
||||
|
||||
@@ -75,7 +75,8 @@ Three properties worth knowing:
|
||||
| [MTConnect](MTConnect.md) | `Driver.MTConnect` (+ `.Contracts`) | — | Hand-rolled `System.Xml.Linq` HTTP/XML client against a vendor-neutral MTConnect Agent (`/probe`, `/current`, `/sample`) — no TrakHound dependency (dropped; see the driver doc) | IDriver, ITagDiscovery, IReadable, ISubscribable, IHostConnectivityProbe, IRediscoverable | Read-only by design (no `IWritable`). Production data plane is entirely `ISubscribable`/`OnDataChange` — `IReadable` is CLI/test-only. Browse is free via the Wave-0 universal discovery browser, no bespoke browser project. `IRediscoverable`/`IHostConnectivityProbe` have no server-side consumer yet — a pre-existing fleet-wide gap, not MTConnect-specific |
|
||||
| [Sql](Sql.md) | `Driver.Sql` (+ `.Browser`, `.Contracts`) | A | `Microsoft.Data.SqlClient` behind an `ISqlDialect` seam (`SqlServerDialect` shipped; Postgres/ODBC deferred) | IDriver, ITagDiscovery, IReadable, ISubscribable, IHostConnectivityProbe | Read-only DB-poll driver with a bespoke schema browser. **Credentials are never persisted** — the deploy gate rejects a stored `connectionString` (#498) and catalog identifiers are validated against the live catalog's own spelling (#496). `SupportsOnlineDiscovery => false` by design (the bespoke browser wins). `SqlTagModel.Query` is deferred to P3 and rejected end-to-end by parser, validator, planner and editor. Merged `4ad54037` |
|
||||
| [Calculation](Calculation.md) | `Driver.Calculation` | A | none — computes from other tags' RawPaths | IDriver, IDependencyConsumer, ISubscribable, IReadable | **Pseudo-driver**, not a wire protocol: signal-level tags computed from other tags via a script, with deploy-time `scriptId`-existence + Tarjan cycle gates. Its probe is parse-only (no backend to reach). `runTimeoutMs` is authorable through `CalculationDriverForm` (the "no typed config form" gap — G-1 — was closed 2026-07-27) |
|
||||
| [Historian.Gateway](../Historian.md) | `Driver.Historian.Gateway` | — | `ZB.MOM.WW.HistorianGateway.Client` gRPC (`historian_gateway.v1`) | IHistorianDataSource (server-side read backend) + alarm `SendEvent` writer + `WriteLiveValues` recorder + `IHistorianProvisioning` | Not a tag driver — the sole historian backend. Registers `GatewayHistorianDataSource : IHistorianDataSource` for HistoryRead and serves alarm-write + continuous historization through the gateway. No `IDriver`/`ITagDiscovery` surface. (The retired Wonderware sidecar backend it replaced is documented at [Historian.Wonderware.md](Historian.Wonderware.md).) |
|
||||
| [Historian.Gateway](../Historian.md) | `Driver.Historian.Gateway` | — | `ZB.MOM.WW.HistorianGateway.Client` gRPC (`historian_gateway.v1`) | IHistorianDataSource (server-side read backend) + alarm `SendEvent` writer + `WriteLiveValues` recorder + `IHistorianProvisioning` | Not a tag driver — the default historian backend (the other is `Historian.SqlServer`, below). Registers `GatewayHistorianDataSource : IHistorianDataSource` for HistoryRead and serves alarm-write + continuous historization through the gateway. No `IDriver`/`ITagDiscovery` surface. (The retired Wonderware sidecar backend it replaced is documented at [Historian.Wonderware.md](Historian.Wonderware.md).) |
|
||||
| [Historian.SqlServer](../Historian.md#sql-server-backend) | `Driver.Historian.SqlServer` | — | `Microsoft.Data.SqlClient` (a plain SQL Server database, schema `hist` by default) | `IHistorianDataSource` + `IHistorianValueWriter` + `IAlarmHistorianWriter` + `IHistorianProvisioning` + `IHistorianDataUpdater` | Not a tag driver — a pluggable historian backend selected with `ServerHistorian:Backend=SqlServer`. Stores typed values with their real status, serves Raw / AtTime / Events / 13 push-down aggregates, and is the only backend that supports OPC UA **HistoryUpdate**. No `IDriver`/`ITagDiscovery` surface. |
|
||||
|
||||
## Per-driver documentation
|
||||
|
||||
@@ -99,8 +100,8 @@ Three properties worth knowing:
|
||||
data-plane precedence rules, and a non-trivial known-limitations list need explaining up front:
|
||||
- [MTConnect.md](MTConnect.md) — Agent config, `FullName`==DataItem `id`, `UNAVAILABLE`→`BadNoCommunication`, CONDITION-as-`String`, browse-via-universal-browser, fixture recipe, deferred write-back
|
||||
|
||||
- **Historian.Gateway** (server-side historian backend, not a tag driver) is documented in the main guide:
|
||||
- [../Historian.md](../Historian.md) — HistorianGateway backend: read-path registration, HistoryRead dispatch, alarm store-and-forward (`SendEvent`), continuous historization (`WriteLiveValues`), `EnsureTags` provisioning, config keys, deployment prerequisites. (The retired Wonderware sidecar backend it replaced: [Historian.Wonderware.md](Historian.Wonderware.md).)
|
||||
- **Historian.Gateway** and **Historian.SqlServer** (server-side historian backends, not tag drivers) are documented in the main guide:
|
||||
- [../Historian.md](../Historian.md) — backend selection (`ServerHistorian:Backend`), the SQL Server backend and HistoryUpdate, plus the HistorianGateway backend: read-path registration, HistoryRead dispatch, alarm store-and-forward (`SendEvent`), continuous historization (`WriteLiveValues`), `EnsureTags` provisioning, config keys, deployment prerequisites. (The retired Wonderware sidecar backend it replaced: [Historian.Wonderware.md](Historian.Wonderware.md).)
|
||||
|
||||
- The full per-field spec (capability surface, config schema, addressing, data-type maps, connection settings, quirks for every driver) lives in [docs/v2/driver-specs.md](../v2/driver-specs.md). The overview pages above are the short path and are the current reference; that file is the v2-era per-field spec (see its status banner for which drivers it covers).
|
||||
|
||||
@@ -121,7 +122,7 @@ Each driver has a dedicated fixture doc that lays out what the integration / uni
|
||||
|
||||
## Related cross-driver docs
|
||||
|
||||
- [HistoricalDataAccess.md](../v1/HistoricalDataAccess.md) — v1 archive of the retired per-driver history design (aggregate mapping, continuation points). History is now served only by the server-side `IHistorianDataSource` (HistorianGateway); no driver implements a history capability, and nodes without `isHistorized` return `BadHistoryOperationUnsupported`.
|
||||
- [HistoricalDataAccess.md](../v1/HistoricalDataAccess.md) — v1 archive of the retired per-driver history design (aggregate mapping, continuation points). History is now served only by the server-side `IHistorianDataSource` (the selected historian backend: HistorianGateway or SQL Server); no driver implements a history capability, and nodes without `isHistorized` return `BadHistoryOperationUnsupported`.
|
||||
- [AlarmTracking.md](../AlarmTracking.md) — `IAlarmSource` event model and filtering. Implemented by Galaxy (native MxAccess alarms, working end-to-end), OPC UA Client, AB CIP, and FOCAS; AB Legacy, Modbus, S7, TwinCAT, and MQTT have no alarm source.
|
||||
- [Subscriptions.md](../v1/Subscriptions.md) — how the Server multiplexes subscriptions onto `ISubscribable.OnDataChange`.
|
||||
- [docs/v2/driver-stability.md](../v2/driver-stability.md) — historical design record of the A / B / C tier model. Only `DriverTier` + the `CapabilityPolicy` defaults per tier × capability (`DriverResilienceOptions.GetTierDefaults`) are live; its `MemoryTracking` / recycle / out-of-process isolation machinery was deleted (#522) and every driver runs Tier A.
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
-- OtOpcUa SQL Server historian backend - schema version 1.
|
||||
-- $(Schema) is substituted by SqlServerSchemaBootstrapper (identifier-validated) BEFORE the script is split on GO lines.
|
||||
-- Every statement is guarded so a re-run is a no-op.
|
||||
IF SCHEMA_ID(N'$(Schema)') IS NULL EXEC(N'CREATE SCHEMA [$(Schema)]')
|
||||
GO
|
||||
IF OBJECT_ID(N'[$(Schema)].[SchemaVersion]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[SchemaVersion] (
|
||||
Version int NOT NULL PRIMARY KEY,
|
||||
AppliedUtc datetime2(7) NOT NULL)
|
||||
GO
|
||||
IF OBJECT_ID(N'[$(Schema)].[Tag]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[Tag] (
|
||||
TagId int IDENTITY(1,1) NOT NULL PRIMARY KEY,
|
||||
Name nvarchar(400) NOT NULL CONSTRAINT UQ_Tag_Name UNIQUE,
|
||||
DataType tinyint NOT NULL,
|
||||
EngineeringUnit nvarchar(64) NULL,
|
||||
Description nvarchar(400) NULL,
|
||||
CreatedUtc datetime2(7) NOT NULL,
|
||||
UpdatedUtc datetime2(7) NOT NULL)
|
||||
GO
|
||||
IF OBJECT_ID(N'[$(Schema)].[Sample]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[Sample] (
|
||||
TagId int NOT NULL,
|
||||
SourceTimeUtc datetime2(7) NOT NULL,
|
||||
ValueKind tinyint NOT NULL,
|
||||
NumValue float NULL,
|
||||
IntValue bigint NULL,
|
||||
StrValue nvarchar(max) NULL,
|
||||
StatusCode int NOT NULL,
|
||||
ServerTimeUtc datetime2(7) NOT NULL,
|
||||
CONSTRAINT PK_Sample PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc),
|
||||
CONSTRAINT FK_Sample_Tag FOREIGN KEY (TagId) REFERENCES [$(Schema)].[Tag](TagId))
|
||||
GO
|
||||
IF OBJECT_ID(N'[$(Schema)].[Event]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[Event] (
|
||||
EventId nvarchar(64) NOT NULL CONSTRAINT PK_Event PRIMARY KEY NONCLUSTERED,
|
||||
SourceName nvarchar(400) NOT NULL,
|
||||
AlarmId nvarchar(400) NOT NULL,
|
||||
AlarmName nvarchar(400) NOT NULL,
|
||||
AlarmTypeName nvarchar(100) NOT NULL,
|
||||
Severity int NOT NULL,
|
||||
EventKind nvarchar(50) NOT NULL,
|
||||
Message nvarchar(max) NULL,
|
||||
UserName nvarchar(200) NULL,
|
||||
Comment nvarchar(max) NULL,
|
||||
EventTimeUtc datetime2(7) NOT NULL,
|
||||
ReceivedTimeUtc datetime2(7) NOT NULL)
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'CX_Event_Time' AND object_id = OBJECT_ID(N'[$(Schema)].[Event]'))
|
||||
CREATE CLUSTERED INDEX CX_Event_Time ON [$(Schema)].[Event] (EventTimeUtc, EventId)
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_Event_Source_Time' AND object_id = OBJECT_ID(N'[$(Schema)].[Event]'))
|
||||
CREATE INDEX IX_Event_Source_Time ON [$(Schema)].[Event] (SourceName, EventTimeUtc)
|
||||
GO
|
||||
IF TYPE_ID(N'[$(Schema)].[SampleRow]') IS NULL
|
||||
CREATE TYPE [$(Schema)].[SampleRow] AS TABLE (
|
||||
SourceTimeUtc datetime2(7) NOT NULL PRIMARY KEY,
|
||||
ValueKind tinyint NOT NULL,
|
||||
NumValue float NULL,
|
||||
IntValue bigint NULL,
|
||||
StrValue nvarchar(max) NULL,
|
||||
StatusCode int NOT NULL)
|
||||
GO
|
||||
IF TYPE_ID(N'[$(Schema)].[TimeRow]') IS NULL
|
||||
CREATE TYPE [$(Schema)].[TimeRow] AS TABLE (T datetime2(7) NOT NULL PRIMARY KEY)
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM [$(Schema)].[SchemaVersion] WHERE Version = 1)
|
||||
INSERT [$(Schema)].[SchemaVersion] (Version, AppliedUtc) VALUES (1, SYSUTCDATETIME())
|
||||
GO
|
||||
@@ -0,0 +1,29 @@
|
||||
-- OtOpcUa SQL Server historian backend - schema version 2: OPC UA Part 11 modified-value history (Gitea #668).
|
||||
-- $(Schema) is substituted by SqlServerSchemaBootstrapper (identifier-validated) BEFORE the script is split on GO lines.
|
||||
-- Every statement is guarded so a re-run is a no-op.
|
||||
--
|
||||
-- One row per sample a HistoryUpdate inserted, replaced or deleted, carrying the PRIOR value (all NULL for an
|
||||
-- insert). Rows are written by the SAME statement that changes hist.Sample (OUTPUT ... INTO), so a failed update
|
||||
-- leaves no audit row and a committed one always has its row.
|
||||
-- WARNING: OUTPUT ... INTO forbids, on its target table: enabled triggers, either side of a FOREIGN KEY, and CHECK
|
||||
-- constraints. Do not add any of those to SampleModified, or every HistoryUpdate fails.
|
||||
IF OBJECT_ID(N'[$(Schema)].[SampleModified]', N'U') IS NULL
|
||||
CREATE TABLE [$(Schema)].[SampleModified] (
|
||||
TagId int NOT NULL,
|
||||
SourceTimeUtc datetime2(7) NOT NULL,
|
||||
ModifiedId bigint IDENTITY(1,1) NOT NULL,
|
||||
ModificationTimeUtc datetime2(7) NOT NULL,
|
||||
UpdateType tinyint NOT NULL,
|
||||
Principal nvarchar(200) NOT NULL,
|
||||
ApiKeyId nvarchar(200) NULL,
|
||||
ValueKind tinyint NULL,
|
||||
NumValue float NULL,
|
||||
IntValue bigint NULL,
|
||||
StrValue nvarchar(max) NULL,
|
||||
StatusCode int NULL,
|
||||
ServerTimeUtc datetime2(7) NULL,
|
||||
CONSTRAINT PK_SampleModified PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc, ModifiedId))
|
||||
GO
|
||||
IF NOT EXISTS (SELECT 1 FROM [$(Schema)].[SchemaVersion] WHERE Version = 2)
|
||||
INSERT [$(Schema)].[SchemaVersion] (Version, AppliedUtc) VALUES (2, SYSUTCDATETIME())
|
||||
GO
|
||||
@@ -702,3 +702,98 @@ ST14:10 String Native value="Z-33323" status=0x00000000 (
|
||||
Every address reads identically on both transports. `S:4` is the free-running clock, so its value differs read to read on each transport (both Good). `I:0` = -29376 = 0x8D40, and bits `/6` and `/15` read true on both while `/0`, `/1`, `/7` read false — the bit is extracted from the word libplctag read, not defaulted. ab_server has no S/O/I files, so this real-controller read is the only wire evidence; the translation is unit-pinned against a model of the `pccc.c` parser (`LibplctagLegacyNameTests`).
|
||||
|
||||
Caveats: `R6:0.DN` and `R6:0.EN` read `false` on both transports, so this read does not discriminate the control-word bit number (a wrong bit would very likely also read false) — the R bit mapping rests on the native table and the `pccc.c` parser model, not on this wire evidence. Writes: on libplctag every address the translation changes (`S:`/`O:`/`I:`, R/UA status bits) is refused `BadNotSupported` (#659 review), so this fix adds no libplctag write path.
|
||||
|
||||
## hw5 — #566 idle-connection close and a native STRING write on Z-35162 — 2026-10-09 (user-approved)
|
||||
|
||||
Press: Z-35162, CompactLogix 5069-L320ERS2 rev 34.11, `ab://10.209.70.160/1,0`, family CompactLogix, Mac VPN. The user
|
||||
approved exactly two things: (A) one read-only idle-connection session for #566 and (B) native array and STRING writes on
|
||||
logic-unreferenced tags. Harness: `AbCip.IntegrationTests/HardwareGate/` (its own namespace, outside the read-only
|
||||
`Differential` guard), both tests env-gated. No packet capture (`tcpdump` needs sudo); the diagnostics tap
|
||||
(`ConnectionDiagnostics.ExchangeObserver`, now also fed the Forward Open reply so the granted RPIs are visible) and the
|
||||
transport's exception text are the evidence.
|
||||
|
||||
### A. #566 — what a real Logix does with a class-3 connection idle past its timeout: it drops TCP, no CIP status
|
||||
|
||||
`AB_DIFF_ENDPOINT=10.209.70.160:44818 AB_EVID_IDLE=1 AB_EVID_IDLE_SECONDS=60` →
|
||||
`IdleConnectionEvidenceTests` (raw: [hw5-idle566.txt](2026-10-07-native-ab-transport-t21/hw5-idle566.txt)). One private
|
||||
pool, keep-alive pushed out to 1 h so nothing touched the connection, open-attempt budget 2, three Read Tag requests of
|
||||
`test2` (DINT) and nothing else. 10:04:49Z–10:05:54Z UTC.
|
||||
|
||||
| t (s) | Event |
|
||||
|---|---|
|
||||
| 0.26 | Large Forward Open (0x5B) granted: O→T API = T→O API = 2 000 000 µs (the requested RPI, unchanged). Multiplier 1 (×8) ⇒ connection timeout **16 s** |
|
||||
| 0.32 | read 1: general status 0x00 |
|
||||
| 0.32 – 60.33 | idle, no traffic (3.75 × the timeout) |
|
||||
| 60.33 | read 2: **failed in 9 ms on the write**: `IOException: Broken pipe` ← `SocketException(Shutdown)` → `AbTransportException(BadConnection)` → OPC UA `BadCommunicationError` (0x80050000). No CIP reply, no encapsulation status, `ConnectionsClosedByTarget` 0 |
|
||||
| 62.47 | read 3: fresh Large Forward Open granted (new O→T id), read general 0x00 — **recovered on the next request**, open attempts 2/2 |
|
||||
| 62.56 | lease released: Forward Close sent/acked 1/1 |
|
||||
|
||||
- **Answer to #566: neither 0x0107 nor 0x0203.** The controller closed the TCP session; the next request's write hit a dead
|
||||
socket. EPIPE on the *first* write after the idle (not a successful write followed by EOF) means the close was already
|
||||
complete on the Mac's side, which is consistent with an RST (or FIN + RST) from the controller; without a capture the
|
||||
segment type and the exact moment within the 60 s window are not observed — only that it happened after the 16 s
|
||||
timeout and before 60 s.
|
||||
- **The transport handles it correctly as it stands, so no production code changed.** The failed write faults the session
|
||||
(`EncapsulationClient.Fault`, `IsConnected` false), the failing request alone fails `BadConnection` (fast, not an
|
||||
exchange timeout), the driver maps that to `BadCommunicationError` and evicts the runtime / reports `Degraded` for that
|
||||
poll, and the next send reopens through the Forward Open throttle. The 0x0107/0x0203 recycle (#566) stays: it is
|
||||
correct for a target that keeps TCP up, and harmless here (not exercised).
|
||||
- **In production this path needs more than idleness.** The keep-alive fires at half the expiry (8 s), so a connection the
|
||||
driver holds never idles 16 s; the shape above arises after a controller restart / re-download or a network gap.
|
||||
- **Pinned offline:** `CipConnectionTests.A_target_that_resets_an_idle_session_fails_the_next_request_fast_and_the_one_after_reopens`
|
||||
replays the shape (a `FakeEipServer.ResetAllConnections` RST with no request in flight): `BadConnection` in < 4 s,
|
||||
`ConnectionsClosedByTarget` 0, one Forward Open before / two after. Proven able to fail: with `EnsureOpenAsync` made to
|
||||
reuse a non-connected session it goes red (the reopen never happens).
|
||||
|
||||
### B. Native writes: STRING PASS; array NOT RUN (no tag qualifies)
|
||||
|
||||
**STRING — `Program:ES03_TemperatureControl._StrTmp` (PASS, restored byte-exact).** Census: refs 0, extrefs 0, not in the
|
||||
Galaxy map, program not HMI/Wonderware; raw grep of `SN3244.L5X` finds the name only at its declaration (line 69426).
|
||||
`AB_HW_WRITE_GATEWAY=ab://10.209.70.160/1,0 AB_HW_WRITE_APPROVED=1 AB_HW_WRITE_STRING_TAG=Program:ES03_TemperatureControl._StrTmp`
|
||||
→ `StringWriteGateTests`, through the production `AbCipDriver` on the native transport, 10:10:54Z–10:11:29Z UTC (raw:
|
||||
[hw5-string-write.txt](2026-10-07-native-ab-transport-t21/hw5-string-write.txt)). A read-only dry run (`AB_HW_WRITE_DRYRUN=1`)
|
||||
first confirmed the preflight. The raw bytes come from the tap on each Read Tag Fragmented reply.
|
||||
|
||||
| Step | Status | Value | Raw (type, handle, LEN, DATA…) |
|
||||
|---|---|---|---|
|
||||
| snapshot 1 | Good | `ES03_TemperatureControl:Fault` | `A002 CE0F 1D000000 4553…6C74 00…` (92 B) |
|
||||
| snapshot 2 (+30 s) | Good | same | identical to snapshot 1 |
|
||||
| preflight | — | — | built-in STRING (0x02A0 / handle 0x0FCE / 88 B), LEN 29, every byte past LEN zero |
|
||||
| write `OTOPCUA-TEST` | Good (0x00000000) | — | — |
|
||||
| read back | Good | `OTOPCUA-TEST` | `A002 CE0F 0C000000 4F544F504355412D54455354 00…` |
|
||||
| restore original | Good | — | — |
|
||||
| read back | Good | `ES03_TemperatureControl:Fault` | **byte-equal to snapshot 1** |
|
||||
| final snapshot (+2 s) | Good | same | byte-equal to snapshot 1 |
|
||||
|
||||
The original matches the L5X export's value. The write is one unfragmented Write Tag (88 B of data plus the path is far
|
||||
below the 4000-byte request limit of the Large Forward Open).
|
||||
|
||||
**Array — not run; no candidate passes the brief's own rule.** The only logic-unreferenced, Read/Write atomic arrays in the
|
||||
census are `Program:EM01_PowerUnit._SEQ_EP` and `Program:EM02_Clamp._SEQ_EP` (DINT[32]), and the raw grep shows they are
|
||||
**not** unreferenced: aliases `_SEQ_EP01_AutoCycle → _SEQ_EP[1]` (EM01) and `_SEQ_EP01_Jog / _SEQ_EP02_Auto /
|
||||
_SEQ_EP03_Stopping → _SEQ_EP[1..3]` (EM02) are the step registers of the power-unit auto cycle and the clamp jog / auto /
|
||||
stopping sequencers, used by dozens of rungs (`MOV(…,_SEQ_EP02_Auto)`, `EQU(_SEQ_EP01_AutoCycle,300)`…). The census counts
|
||||
references by tag name, so references through an alias were invisible to it. A whole-array write would rewrite live
|
||||
sequencer state, and even a single unaliased element fails the "name appears only at its declaration" check. The other
|
||||
arrays are referenced (`_DisableStates`, `_ModeTransitions`, `_ModeNames`, `_VacuumCalibration_*`, `Stored`,
|
||||
`_monitorID`), `ExternalAccess="Read Only"` (`_SEQ_UP`, `Inp_CycleTime`) or framework lookup tables (`Inf_LookupFwkB`). No
|
||||
array write was made. An element write needs a tag the user picks (or an unaliased element they approve explicitly).
|
||||
|
||||
**#567 (fragmented wide-struct write) — still cannot be done here, and a DINT[32] would not have done it.** AbCip has no
|
||||
whole-array write at the driver layer (array tags are read-only, `DriverArrayWriteSupport`); and at the transport layer a
|
||||
128-byte array (or this 88-byte STRING) is a single Write Tag: `LogixTagClient.WriteAsync` fragments only when the request
|
||||
exceeds `MaxRequestBytes` (4000 B on the Large Forward Open this controller grants). A fragmented write needs a value of
|
||||
roughly 4 KB.
|
||||
|
||||
### Native array-element write — 2026-10-09 (user-approved, Z-35162)
|
||||
|
||||
`Program:EM01_PowerUnit._SEQ_EP` (DINT[32]). L5X check: its only alias is `_SEQ_EP01_AutoCycle -> _SEQ_EP[1]`; no routine references `_SEQ_EP` directly (no index, COP or FLL); no cross-program reference. AbCip CLI (native) from master `98c00904`, one element per read, two snapshots 30 s apart before writing. `[1]` = -1 is the idle auto-cycle step.
|
||||
|
||||
```
|
||||
SNAP1: 0 -1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
SNAP2: 0 -1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
WRITE [31]=4242: Write Program:EM01_PowerUnit._SEQ_EP[31]: 0x00000000 (Good)
|
||||
AFTER: 0 -1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4242
|
||||
RESTORE [31]=0: Write Program:EM01_PowerUnit._SEQ_EP[31]: 0x00000000 (Good)
|
||||
FINAL: 0 -1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
|
||||
```
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# hw5 #566 idle-connection gate, Z-35162, 2026-10-09 10:04:49Z-10:05:54Z UTC
|
||||
# AB_DIFF_ENDPOINT=10.209.70.160:44818 AB_EVID_IDLE=1 AB_EVID_IDLE_SECONDS=60 -> HardwareGate/IdleConnectionEvidenceTests
|
||||
Passed ZB.MOM.WW.OtOpcUa.Driver.AbCip.IntegrationTests.HardwareGate.IdleConnectionEvidenceTests.An_idle_connection_past_its_timeout_reports_what_the_controller_does_and_recovers [1 m 2 s]
|
||||
[ 0.00 s] Target 10.209.70.160:44818 path '1,0' family CompactLogix (compactlogix, connection size 4002), max tags 200, max templates 50, pace 20 ms; tag 'test2'; requested RPI 2000000 us both ways, timeout multiplier 1 (x8) => expiry 16.0 s; keep-alive pushed to 01:00:00; idle 60 s
|
||||
[ 0.26 s] Forward Open 0x5B granted: O->T id 0xFFFE58C1, O->T API 2000000 us, T->O API 2000000 us; raw DB000000C158FEFF067DB1E37B4F3713143E526D80841E0080841E000000
|
||||
[ 0.32 s] exchange: request service 0x4C (12 B) -> reply 10 B: CC000000C40000000000
|
||||
[ 0.32 s] read 1 (opens): reply in 314 ms, general 0x00
|
||||
[ 0.32 s] Forward Opens 1; now idle for 60 s with no traffic
|
||||
[ 60.33 s] read 2 (after idle): THREW after 9 ms: AbTransportException status=BadConnection (OPC UA 0x80050000) message='Connection lost: Unable to write data to the transport connection: Broken pipe.' inner=IOException('Unable to write data to the transport connection: Broken pipe.') <- SocketException(Shutdown)
|
||||
[ 60.33 s] After read 2: Forward Opens 1 (delta 0), closed-by-target 0, success=False
|
||||
[ 62.47 s] Forward Open 0x5B granted: O->T id 0xFFFE58C2, O->T API 2000000 us, T->O API 2000000 us; raw DB000000C258FEFF489E51887C4F3713143E526D80841E0080841E000000
|
||||
[ 62.51 s] exchange: request service 0x4C (12 B) -> reply 10 B: CC000000C40000000000
|
||||
[ 62.51 s] read 3 (recovery): reply in 178 ms, general 0x00
|
||||
[ 62.51 s] After read 3: Forward Opens 2, closed-by-target 0, open attempts used 2/2, success=True
|
||||
[ 62.56 s] Lease released: Forward Closes sent/acked 1/1
|
||||
Total time: 1.0757 Minutes
|
||||
@@ -0,0 +1,15 @@
|
||||
# hw5 native STRING write gate, Z-35162, 2026-10-09 10:10:54Z-10:11:29Z UTC (user-approved; restored)
|
||||
# AB_HW_WRITE_GATEWAY=ab://10.209.70.160/1,0 AB_HW_WRITE_APPROVED=1 AB_HW_WRITE_STRING_TAG=Program:ES03_TemperatureControl._StrTmp -> HardwareGate/StringWriteGateTests
|
||||
Passed ZB.MOM.WW.OtOpcUa.Driver.AbCip.IntegrationTests.HardwareGate.StringWriteGateTests.A_native_STRING_write_reads_back_and_restores_the_exact_original [32 s]
|
||||
Gateway ab://10.209.70.160/1,0; tag 'Program:ES03_TemperatureControl._StrTmp'; transport Native; snapshot gap 30 s
|
||||
snapshot 1: status 0x00000000, value 'ES03_TemperatureControl:Fault', raw (92 B) A002CE0F1D000000455330335F54656D7065726174757265436F6E74726F6C3A4661756C7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
snapshot 2: status 0x00000000, value 'ES03_TemperatureControl:Fault', raw (92 B) A002CE0F1D000000455330335F54656D7065726174757265436F6E74726F6C3A4661756C7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
Preflight OK: type/handle A002CE0F, LEN 29, tail zero; original 'ES03_TemperatureControl:Fault'
|
||||
write test value: 'OTOPCUA-TEST' (ASCII 4F544F504355412D54455354) -> status 0x00000000
|
||||
read back test value: status 0x00000000, value 'OTOPCUA-TEST', raw (92 B) A002CE0F0C0000004F544F504355412D54455354000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
restore original: 'ES03_TemperatureControl:Fault' (ASCII 455330335F54656D7065726174757265436F6E74726F6C3A4661756C74) -> status 0x00000000
|
||||
read back restored: status 0x00000000, value 'ES03_TemperatureControl:Fault', raw (92 B) A002CE0F1D000000455330335F54656D7065726174757265436F6E74726F6C3A4661756C7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
Test-value raw LEN 12; restored raw equals original: True
|
||||
final snapshot: status 0x00000000, value 'ES03_TemperatureControl:Fault', raw (92 B) A002CE0F1D000000455330335F54656D7065726174757265436F6E74726F6C3A4661756C7400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
|
||||
RESULT: write, read-back, exact restore — PASS
|
||||
Total time: 34.8142 Seconds
|
||||
@@ -0,0 +1,368 @@
|
||||
# Pluggable historian backends — design
|
||||
|
||||
**Date:** 2026-10-09 · **Status:** approved (brainstorm 2026-10-09) · **Plan:** `2026-10-09-pluggable-historian-plan.md`
|
||||
|
||||
## 1. Goal and scope
|
||||
|
||||
Today the only historian backend is AVEVA/Wonderware, reached through the HistorianGateway sidecar. This
|
||||
work makes the backend **pluggable**, ships **SQL Server** as the first new backend, and adds the OPC UA
|
||||
**HistoryUpdate** service so clients can insert / replace / update / delete raw history.
|
||||
|
||||
| In scope (this spec) | Later (own spec) | Dropped |
|
||||
|---|---|---|
|
||||
| Provider seam + `ServerHistorian:Backend` selection | QuestDB backend (a new provider only) | SQLite (a per-node file needs pair replication) |
|
||||
| Typed values (not just `double`) end to end through recorder + outbox | | |
|
||||
| Real status codes recorded (fixes a hard-coded Good) | | |
|
||||
| SQL Server backend: schema, writes, reads, aggregates push-down, alarm events, provisioning, updates | | |
|
||||
| HistoryUpdate service + `HistoryUpdate` role / permission + AdminUI checkbox | | |
|
||||
| Client.CLI `historyupdate` command | | |
|
||||
|
||||
Out of scope everywhere: retention/purge, partitioning/compression, Part 11 "modified values"
|
||||
(`IsReadModified`, `IsDeleteModified`), event HistoryUpdate (`UpdateEventDetails`/`DeleteEventDetails`),
|
||||
`UpdateStructureData`, `AuditHistoryUpdateEvent` emission, array historization.
|
||||
|
||||
Hard constraints: no new NuGet packages and no version changes (`Microsoft.Data.SqlClient` is already
|
||||
centrally pinned — referencing it from a new project is allowed, adding a new package is not).
|
||||
Recording stays Primary-only (#537); delivery stays at-least-once with the #538 per-tag quarantine.
|
||||
|
||||
## 2. Architecture — the provider seam (approach A)
|
||||
|
||||
The five historian seams the rest of the code already consumes stay; a provider builds them.
|
||||
|
||||
```csharp
|
||||
// src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/IHistorianBackendProvider.cs
|
||||
public interface IHistorianBackendProvider
|
||||
{
|
||||
string Name { get; } // "Gateway" | "SqlServer" | (later) "QuestDb"
|
||||
IReadOnlySet<HistorianValueKind> SupportedKinds { get; } // recorder capture filter
|
||||
IHistorianDataSource CreateDataSource(IServiceProvider sp);
|
||||
IHistorianValueWriter CreateValueWriter(IServiceProvider sp);
|
||||
IAlarmHistorianWriter CreateAlarmWriter(IServiceProvider sp);
|
||||
IHistorianProvisioning CreateProvisioning(IServiceProvider sp);
|
||||
IHistorianDataUpdater? CreateUpdater(IServiceProvider sp); // null ⇒ HistoryUpdate unsupported
|
||||
}
|
||||
```
|
||||
|
||||
It lives in **Runtime** because the seams it returns span `Core.Abstractions` and `Core.AlarmHistorian`.
|
||||
|
||||
- `GatewayHistorianProvider` (Driver.Historian.Gateway) wraps today's `GatewayHistorian.Create…` factories;
|
||||
`CreateUpdater` returns `null`; `SupportedKinds = {Double, Int64, Boolean}`.
|
||||
- `SqlServerHistorianProvider` (new project `Driver.Historian.SqlServer`); `SupportedKinds` = all kinds.
|
||||
- The Host owns a name→provider table (`HistorianBackendRegistry`). `Program.cs`'s four hard-wired
|
||||
`GatewayHistorian.Create…` call sites (three, plus the inline `new GatewayHistorianValueWriter(...)`) become calls
|
||||
on the selected provider. The existing gates (`AddServerHistorian`, `AddHistorianProvisioning`,
|
||||
`AddAlarmHistorian`, the outbox registration, the recorder spawn) are unchanged — they get their objects
|
||||
from the provider.
|
||||
- The outbox (`FasterLogHistorizationOutbox`) stays in the Gateway project (moving it would add a
|
||||
FASTER package reference elsewhere); the Host registers it regardless of backend. The SqlServer project
|
||||
does **not** reference the Gateway project.
|
||||
- Rejected: one fat `IHistorian` (forces stubs, churns every narrow consumer); per-capability backend
|
||||
selection (YAGNI, doubles config + validation).
|
||||
|
||||
### Configuration
|
||||
|
||||
```jsonc
|
||||
"ServerHistorian": {
|
||||
"Enabled": true,
|
||||
"Backend": "SqlServer", // "Gateway" (default when absent) | "SqlServer"
|
||||
"HistoryReadBatchParallelism": 4, "MaxConcurrentHistoryReadBatches": 16, "HistoryReadDeadline": "00:01:00",
|
||||
"Endpoint": "...", "ApiKey": "...", // gateway keys stay top-level, unchanged (no migration)
|
||||
"SqlServer": {
|
||||
"ConnectionString": "${secret:OT-HISTORIAN-SQL}",
|
||||
"Schema": "hist",
|
||||
"CommandTimeout": "00:00:30",
|
||||
"AutoCreateSchema": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- Unknown `Backend` ⇒ host start fails (`OptionsValidationException` listing the valid names).
|
||||
- `ServerHistorianOptionsValidator` runs its three gateway checks (Endpoint, ApiKey, UseTls⇔scheme) only when `Backend == Gateway`; backend-specific options are validated there too, consumer-gated (`ServerHistorian:Enabled || AlarmHistorian:Enabled`), never at registry resolution.
|
||||
- New `SqlServerHistorianOptions` validation: `ConnectionString` required, `Schema` a valid SQL identifier
|
||||
(`^[A-Za-z_][A-Za-z0-9_]{0,127}$` — it is spliced into DDL/DML), `CommandTimeout > 0`.
|
||||
- Validators remain driver-block-only (admin-only nodes never build historian singletons). *As built:* an invalid `SqlServer:Schema`
|
||||
also throws `ArgumentException` when the provider is constructed, so `Backend=SqlServer` + a malformed schema stops host start even
|
||||
when no consumer is enabled.
|
||||
- The `Program.cs` alarm-only warning names the selected backend instead of "gateway".
|
||||
- The historian connection string is **not** a ConfigDb string; driver-only nodes may carry it (Phase 4 intact).
|
||||
|
||||
## 3. Typed values
|
||||
|
||||
```csharp
|
||||
// Core.Abstractions/Historian/HistorianValue.cs
|
||||
public enum HistorianValueKind : byte { Empty = 0, Double = 1, Int64 = 2, Boolean = 3, String = 4, DateTime = 5 }
|
||||
|
||||
public readonly record struct HistorianValue(HistorianValueKind Kind, double Number, long Integer, string? Text)
|
||||
{
|
||||
public static HistorianValue Empty { get; } // no value (a Bad/Uncertain sample with a null)
|
||||
public static HistorianValue FromDouble(double v);
|
||||
public static HistorianValue FromInt64(long v);
|
||||
public static HistorianValue FromBoolean(bool v); // Integer = 1/0, Number = 1/0
|
||||
public static HistorianValue FromString(string v);
|
||||
public static HistorianValue FromDateTime(DateTime utc); // Integer = Ticks (UTC)
|
||||
public static bool TryFromObject(object? value, out HistorianValue result); // the recorder's coercion
|
||||
public bool TryGetDouble(out double v); // Double/Int64/Boolean ⇒ true
|
||||
public object? ToObject(); // back to a CLR value for DataValueSnapshot
|
||||
}
|
||||
```
|
||||
|
||||
`TryFromObject` mapping: `double/float/decimal` ⇒ Double; `sbyte/byte/short/ushort/int/uint/long` ⇒ Int64;
|
||||
`ulong` ⇒ Int64 when `≤ long.MaxValue` else Double; `bool` ⇒ Boolean; `string` ⇒ String; `DateTime` ⇒
|
||||
DateTime (converted to UTC); `null` ⇒ Empty; arrays / anything else ⇒ `false` (dropped + metered).
|
||||
|
||||
Changed records:
|
||||
|
||||
```csharp
|
||||
public readonly record struct HistorizationValue(DateTime? TimestampUtc, HistorianValue Value, uint StatusCode);
|
||||
public sealed record HistorizationOutboxEntry(Guid Id, string Tag, HistorianValue Value, uint StatusCode, DateTime TimestampUtc);
|
||||
```
|
||||
|
||||
**Recorder.** Captures `HistorianValue` via `TryFromObject`; records `msg.StatusCode` (the mux already
|
||||
carries it — the recorder hard-codes `GoodQuality = 192` today, so Bad samples were historized as Good).
|
||||
A value whose kind is not in the provider's `SupportedKinds` is dropped at capture and counted under
|
||||
`DroppedUnsupportedKind` (renamed from `DroppedNonNumeric`, same meter semantics for the gateway). `Empty`
|
||||
is in SQL Server's set (a Bad sample with no value is history worth keeping) and not the gateway's.
|
||||
|
||||
**Gateway backend: no wire change in practice.** Samples now carry their real status, mapped to DA quality
|
||||
inside `GatewayHistorianValueWriter` by a new reverse mapper `GatewayQualityMapper.ToDaQuality(uint)` — the
|
||||
exact inverse of the existing `Map(byte)` table, falling back by severity (Good ⇒ 192, Uncertain ⇒ 64,
|
||||
Bad ⇒ 0). The recorder no longer knows about DA quality. But the gateway's `WriteLiveValues` SQL path
|
||||
**discards** `Quality` (Contracts C-002, server-managed), so AVEVA keeps storing every recorded value as Good —
|
||||
a pre-existing limitation of that backend, now documented rather than fixed. The SQL Server backend stores
|
||||
the real status.
|
||||
|
||||
**Outbox format v2.** Entries are written `magic "HV2\0"` + `version (1 byte)` + id(16) + tagLen(4) + tag +
|
||||
kind(1) + kind payload + status(4) + timestamp(8). Payload by kind: Empty none; Double 8; Int64/Boolean/DateTime
|
||||
8; String len(4)+UTF-8. Legacy entries (no magic, `length == 38 + tagLen@16`) are read as Double with
|
||||
status `GatewayQualityMapper.Map(quality)` (DA ⇒ StatusCode; legacy entries are always 192 ⇒ Good). An entry
|
||||
matching neither layout is dropped and counted (`otopcua.historian.outbox.unreadable`), never thrown — a
|
||||
corrupt entry must not jam the drain. A pinned legacy-bytes test guards the upgrade path.
|
||||
|
||||
## 4. SQL Server backend
|
||||
|
||||
### Schema (`{Schema}` default `hist`, database e.g. `OtOpcUaHistorian`)
|
||||
|
||||
```sql
|
||||
CREATE TABLE {s}.SchemaVersion (Version int NOT NULL PRIMARY KEY, AppliedUtc datetime2(7) NOT NULL);
|
||||
|
||||
CREATE TABLE {s}.Tag (
|
||||
TagId int IDENTITY(1,1) NOT NULL PRIMARY KEY,
|
||||
Name nvarchar(400) NOT NULL CONSTRAINT UQ_Tag_Name UNIQUE,
|
||||
DataType tinyint NOT NULL, -- DriverDataType
|
||||
EngineeringUnit nvarchar(64) NULL, Description nvarchar(400) NULL,
|
||||
CreatedUtc datetime2(7) NOT NULL, UpdatedUtc datetime2(7) NOT NULL);
|
||||
|
||||
CREATE TABLE {s}.Sample (
|
||||
TagId int NOT NULL, SourceTimeUtc datetime2(7) NOT NULL,
|
||||
ValueKind tinyint NOT NULL, -- HistorianValueKind
|
||||
NumValue float NULL, IntValue bigint NULL, StrValue nvarchar(max) NULL,
|
||||
StatusCode int NOT NULL, -- uint StatusCode stored bit-for-bit (unchecked cast)
|
||||
ServerTimeUtc datetime2(7) NOT NULL,
|
||||
CONSTRAINT PK_Sample PRIMARY KEY CLUSTERED (TagId, SourceTimeUtc),
|
||||
CONSTRAINT FK_Sample_Tag FOREIGN KEY (TagId) REFERENCES {s}.Tag(TagId));
|
||||
|
||||
CREATE TABLE {s}.Event (
|
||||
EventId nvarchar(64) NOT NULL CONSTRAINT PK_Event PRIMARY KEY NONCLUSTERED, -- deterministic SHA-256 hex of the event payload
|
||||
SourceName nvarchar(400) NOT NULL, -- AlarmHistorianEvent.EquipmentPath
|
||||
AlarmId nvarchar(400) NOT NULL, AlarmName nvarchar(400) NOT NULL, AlarmTypeName nvarchar(100) NOT NULL,
|
||||
Severity int NOT NULL, EventKind nvarchar(50) NOT NULL, Message nvarchar(max) NULL,
|
||||
UserName nvarchar(200) NULL, Comment nvarchar(max) NULL,
|
||||
EventTimeUtc datetime2(7) NOT NULL, ReceivedTimeUtc datetime2(7) NOT NULL);
|
||||
CREATE CLUSTERED INDEX CX_Event_Time ON {s}.Event (EventTimeUtc, EventId); -- as built (§9.6); the sketched IX_Event_Time is replaced by this
|
||||
CREATE INDEX IX_Event_Source_Time ON {s}.Event (SourceName, EventTimeUtc);
|
||||
|
||||
CREATE TYPE {s}.SampleRow AS TABLE (
|
||||
SourceTimeUtc datetime2(7) NOT NULL PRIMARY KEY, ValueKind tinyint NOT NULL,
|
||||
NumValue float NULL, IntValue bigint NULL, StrValue nvarchar(max) NULL, StatusCode int NOT NULL);
|
||||
CREATE TYPE {s}.TimeRow AS TABLE (T datetime2(7) NOT NULL PRIMARY KEY);
|
||||
```
|
||||
|
||||
Column mapping per kind: Double ⇒ NumValue; Int64 ⇒ IntValue + NumValue (double, for aggregates);
|
||||
Boolean ⇒ IntValue 0/1 + NumValue 0/1; String ⇒ StrValue; DateTime ⇒ IntValue (ticks); Empty ⇒ all NULL.
|
||||
|
||||
The PK enforces Part 11's one-value-per-timestamp and makes outbox redelivery idempotent: live writes are
|
||||
**insert-if-absent**. Events are insert-if-absent on `EventId` (`AlarmHistorianEvent` carries no id, so
|
||||
the writer computes `SHA-256(AlarmId|EventKind|TimestampUtc.Ticks|User|Comment|Message)`, hex).
|
||||
|
||||
**Bootstrap.** Numbered idempotent scripts (`001_initial.sql`) are embedded resources (and mirrored to
|
||||
`docs/historian/sqlserver/`). `SqlServerSchemaBootstrapper` runs once per process on first use, lazily,
|
||||
under `sp_getapplock @Resource='OtOpcUaHistorianSchema', @LockMode='Exclusive', @LockOwner='Session'`.
|
||||
`AutoCreateSchema=true` creates the schema + applies missing scripts; `false` only checks
|
||||
`SchemaVersion` and, if behind, every historian call fails with a clear error naming the missing version
|
||||
(surfaced in `GetHealthSnapshot().LastError`, never a host crash). Bootstrap failure is retried on the
|
||||
next call (not latched).
|
||||
|
||||
### Writes
|
||||
|
||||
- `IHistorianValueWriter.WriteLiveValuesAsync(tag, values)`: resolve `TagId` via a
|
||||
`ConcurrentDictionary<string,int>` cache, creating the tag row on a miss (`MERGE` on Name; DataType from
|
||||
the first value's kind, provisioning later corrects it). Insert via `{s}.SampleRow` TVP with
|
||||
`INSERT … SELECT … WHERE NOT EXISTS`. Returns `true` on success, `false` on a SQL failure (the recorder's
|
||||
retry/quarantine owns retry). `ServerTimeUtc = SYSUTCDATETIME()`. *As built:* duplicate timestamps within one batch
|
||||
collapse to the **first**; a `NaN`/`±Inf` `Double` is stored as `Empty` (it cannot live in a SQL `float`) and, per the
|
||||
final-review I-1 ruling, **an `Empty` sample is always Bad**: a non-Bad non-finite value is stored `BadOutOfRange`, any
|
||||
other non-Bad `Empty` `BadNoData`, an existing Bad code kept; reads clamp a legacy non-Bad `Empty` row to `BadNoData`;
|
||||
tag names over 400 characters are refused, never truncated; a 2627/2601 duplicate-key error rolls the statement back, so the statement is re-run (≤ 2 retries) rather
|
||||
than treated as success.
|
||||
- `IHistorianProvisioning.EnsureTagsAsync`: `MERGE` upsert of every request (all types — no numeric
|
||||
filter on this backend). Per-request failures are counted, never thrown.
|
||||
- `IAlarmHistorianWriter.WriteBatchAsync`: per event insert-if-absent; outcome `Ack` on insert or
|
||||
already-present, `RetryPlease` on transient SQL errors (connection/timeout/deadlock 1205), `PermanentFail`
|
||||
on data errors (e.g. truncation 2628/8152).
|
||||
|
||||
### Reads
|
||||
|
||||
- **Raw:** `SELECT TOP(@max) … WHERE TagId=@id AND SourceTimeUtc >= @lo AND SourceTimeUtc < @hi ORDER BY
|
||||
SourceTimeUtc ASC` (forward) or `> @lo AND <= @hi … DESC` when `endUtc < startUtc` (Part 11 reverse).
|
||||
Unknown tag ⇒ empty result (node manager reports `GoodNoData`). The node manager's paging is unchanged;
|
||||
tie clusters cannot occur (PK).
|
||||
- **AtTime:** one statement, `{s}.TimeRow` TVP + two `OUTER APPLY`s (previous-or-equal, next). Exact ⇒ stored
|
||||
value + status. Otherwise Double/Int64 linearly interpolated (status `Good` with the historian
|
||||
*Interpolated* info bits **when both neighbours are Good, else `UncertainDataSubNormal` + Interpolated**; Int64 is
|
||||
interpolated through `double` and rounded, ties away from zero), Boolean/String/DateTime — and any time **after the last
|
||||
sample** — stepped from the previous sample (status keeps the previous sample's code + Interpolated bit); no previous
|
||||
sample ⇒ `BadNoData`.
|
||||
If the **previous** neighbour is Bad, the requested time lies inside an outage: the result is a null value
|
||||
carrying that neighbour's Bad code (never better than its source, per the quality contract; corrected after the
|
||||
T10 review). If the previous neighbour is usable and the **next** is Bad, the previous value is held (stepped,
|
||||
not interpolated into the outage) with `UncertainDataSubNormal`.
|
||||
- **Events:** `TOP(@maxEvents)` from `{s}.Event` filtered by `SourceName = @source` (null ⇒ all) and
|
||||
`EventTimeUtc` range, ordered ascending (a reversed window is swapped and still returned ascending). `HistoricalEvent(EventId, SourceName, EventTimeUtc,
|
||||
ReceivedTimeUtc, Message, (ushort)Severity)`.
|
||||
- **Processed (push-down):** bucket `b = elapsedTicks / @intervalTicks` in **100 ns ticks** (*corrected after T10: the original
|
||||
sketch used `DATEDIFF_BIG(millisecond, …) / @intervalMs`; `DATEDIFF` counts boundaries crossed, so a ms formulation
|
||||
misplaces samples near a bucket edge and a ns one overflows past ≈292 years. As built:
|
||||
`DATEDIFF_BIG(microsecond, @start, t) × 10` plus a sub-microsecond tick correction*), one row per bucket start `@start + b × interval` for every interval in `[start, end)` (empty intervals
|
||||
emitted as `BadNoData`). Usable sample = status severity bits Good (`00`) or Uncertain (`01`) **and a stored value** (an `Empty`
|
||||
outage marker is Bad whatever its code). Every result carries the historian *Calculated* info bits. An interval with any
|
||||
**non-Good (Bad or Uncertain)** sample but ≥1 usable ⇒ `UncertainDataSubNormal`; no usable ⇒ `BadNoData` (so `Count` over an
|
||||
all-Bad interval is `BadNoData`, not 0). A reversed window is swapped, anchored at the earlier time, and returned descending.
|
||||
|
||||
| Aggregate | SQL over usable samples in the bucket | Value timestamp |
|
||||
|---|---|---|
|
||||
| Average / Minimum / Maximum / Total / Count | `AVG / MIN / MAX / SUM (NumValue)`, `COUNT(*)` | bucket start |
|
||||
| Start / End | first / last by `SourceTimeUtc` (`ROW_NUMBER`) | bucket start |
|
||||
| Delta | End − Start | bucket start |
|
||||
| Range | Max − Min | bucket start |
|
||||
| MinimumActualTime / MaximumActualTime | min / max value, ties ⇒ earliest | the sample's time |
|
||||
| PercentGood / PercentBad | `100 × count(Good) / count(*)` and `100 × count(Bad) / count(*)` over ALL samples; **Uncertain samples are neither** (so the two can sum to < 100) | bucket start |
|
||||
|
||||
Count / Start / End / PercentGood / PercentBad accept any kind; the rest on a String/DateTime tag ⇒
|
||||
`BadAggregateInvalidInputs` per interval. Any other `HistoryAggregateType` ⇒ throw
|
||||
`HistoryAggregateNotSupportedException` (⇒ `BadAggregateNotSupported`). These are the simple per-sample
|
||||
definitions, not Part 13's full interval-boundary semantics — documented as such.
|
||||
|
||||
### Updates (`IHistorianDataUpdater`)
|
||||
|
||||
```csharp
|
||||
public enum HistoryUpdateMode { Insert, Replace, Update } // Part 11 PerformUpdateType minus Remove
|
||||
public interface IHistorianDataUpdater
|
||||
{
|
||||
Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, CancellationToken ct); // one code per value, in order
|
||||
Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken ct);
|
||||
Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, CancellationToken ct);
|
||||
}
|
||||
```
|
||||
|
||||
| Operation | Per-item result |
|
||||
|---|---|
|
||||
| Insert | `GoodEntryInserted`; `BadEntryExists` if a sample exists at that timestamp |
|
||||
| Replace | `GoodEntryReplaced`; `BadNoEntryExists` if none |
|
||||
| Update | `GoodEntryInserted` or `GoodEntryReplaced` |
|
||||
| DeleteRaw | `Good`; `BadNoData` if the range held nothing |
|
||||
| DeleteAtTime | per time: `Good`; `BadNoEntryExists` |
|
||||
|
||||
Each call is one transaction (`MERGE … OUTPUT $action` against the TVP). A value with a null timestamp ⇒
|
||||
`BadInvalidTimestamp` for that item. *As built:* a `NaN`/`±Inf` `Double` ⇒ `BadInvalidArgument` (rejected, not stored as `Empty`
|
||||
as the live-value writer does), and so is a null value with a non-Bad status (I-1); no DataType check against the node (M-1,
|
||||
decided: values are stored as sent); a `DeleteRawModified` with a `MinValue` bound ⇒ `BadInvalidArgument`; duplicate timestamps in one request — Insert: first applies, later `BadEntryExists`; Replace/Update:
|
||||
last wins, earlier ones share its code; DeleteAtTime: first reports, later `BadNoEntryExists`. The node manager answers a null item
|
||||
`BadInvalidArgument`, `MinValue` `BadInvalidTimestamp`, an untypable value `BadTypeMismatch`, a backend throw `BadInternalError`
|
||||
and a deadline `BadTimeout` (a failure is not "unsupported"). Update/Insert on an unknown tag creates the tag row (same as live
|
||||
writes); Replace/Delete on an unknown tag ⇒ `BadNoEntryExists` / `BadNoData`.
|
||||
|
||||
**Status codes.** Every status this backend produces is a named `const uint` in an internal
|
||||
`SqlHistorianStatusCodes` class (the house rule: `StatusCodeParityTests` guards named constants by reflection
|
||||
against `Opc.Ua.StatusCodes`, and an inline literal is invisible to it) — register the class with that parity
|
||||
test. Historian info bits follow Part 4 §7.39: `InfoType = DataValue (0x00000400)` OR-ed with
|
||||
`HistorianCalculated (0x1)` / `HistorianInterpolated (0x2)`; the constants are checked against the SDK's
|
||||
`AggregateBits`/`StatusCode` helpers in a unit test rather than trusted from this doc.
|
||||
|
||||
## 5. HistoryUpdate in the server
|
||||
|
||||
- `OtOpcUaNodeManager` overrides the SDK's protected `HistoryUpdateData`, `HistoryDeleteRawModified`,
|
||||
`HistoryDeleteAtTime` (verify exact signatures against the pinned SDK before writing; the read overrides
|
||||
at `OtOpcUaNodeManager.cs` ~4928 are the pattern). The node → historian tagname lookup is the existing
|
||||
`TryGetHistorizedTagname`, so raw and UNS NodeIds hit the same rows. Not historized ⇒
|
||||
`BadHistoryOperationUnsupported`.
|
||||
- New settable `IHistorianDataUpdater? HistorianDataUpdater` on the node manager, assigned by
|
||||
`OtOpcUaServerHostedService` next to `HistorianDataSource` (resolved as optional — `null` when the
|
||||
backend has none). Null ⇒ every HistoryUpdate item `BadHistoryOperationUnsupported`.
|
||||
- AccessLevel: `ComposeAccessLevel(writable, historized)` gains `historyWritable` — the `HistoryWrite` bit
|
||||
is OR-ed in when historized **and** an updater is wired (both raw node and UNS mirrors).
|
||||
- Out-of-scope details (events, structure data, `IsDeleteModified`) ⇒ `BadHistoryOperationUnsupported`.
|
||||
- Updates go straight to the backend (not the outbox), are not Primary-gated (the database is shared),
|
||||
and share the HistoryRead batch limiter + deadline.
|
||||
- **Authorization.**
|
||||
- ACL flag **on**: `DataPlaneAuthorizer.Authorize(identity, OpcUaOperation.HistoryUpdate, nodeId, realm)`
|
||||
(`TriePermissionEvaluator` already maps it to `NodePermissions.HistoryUpdate`).
|
||||
- ACL flag **off**: new role `OpcUaDataPlaneRoles.HistoryUpdate = "HistoryUpdate"`, conferred by
|
||||
`ApiKeyInterimRoleMapper` only from a **Cluster-scoped** grant carrying `NodePermissions.HistoryUpdate`,
|
||||
plus `AuthorizeKeyUsable` (#563 key re-check). Deny ⇒ `BadUserAccessDenied`.
|
||||
- **Load-bearing test:** an ALLOW — flag on, session with no roles, scoped `HistoryUpdate` grant ⇒ the
|
||||
insert lands. And flag off: a `WriteOperate`-only key is DENIED (the role is not reused).
|
||||
- `NodePermissions.HistoryUpdate` is added to the AdminUI ACL editor's permission list (`AclEdit.razor`);
|
||||
it stays out of every composite bundle.
|
||||
|
||||
## 6. Client.CLI
|
||||
|
||||
`historyupdate` command: `-u <url> -n <nodeId> --mode insert|replace|update|delete-raw|delete-at-time`,
|
||||
`--value <v> --time <iso>` (repeatable for insert/replace/update/delete-at-time), `--start/--end` for
|
||||
delete-raw. Prints one line per item with its status code name. Docs in `docs/Client.CLI.md`.
|
||||
|
||||
## 7. Testing
|
||||
|
||||
- **Unit:** `HistorianValue` coercion table; outbox v2 round-trip per kind + pinned legacy bytes +
|
||||
unreadable-entry drop; recorder typed capture, real status passthrough, `SupportedKinds` filter (incl.
|
||||
`Empty`), unchanged #537/#538 behaviour; gateway writer conversion (Good bytes identical to today);
|
||||
provider registry + validators; role mapper; node manager HistoryUpdate (fake updater): allow/deny matrix
|
||||
for flag on/off, result-code passthrough, unsupported cases, AccessLevel bit.
|
||||
- **SQL Server integration** (`Driver.Historian.SqlServer.IntegrationTests`): env-gated on
|
||||
`HISTORIAN_SQL_TEST_CONNECTION` (a master-capable connection string; absent ⇒ `Assert.Skip`), each run in
|
||||
a throwaway database `OtOpcUaHistorian_T_<guid>` dropped on dispose. Default target: the docker-dev
|
||||
`sql` container, `Server=localhost,14330;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True`.
|
||||
Covers bootstrap (idempotent; two concurrent bootstrappers), every kind round-trip, redelivery idempotency,
|
||||
raw forward/reverse/limit, AtTime exact/interpolated/stepped/no-prior, each aggregate against hand-computed
|
||||
values incl. Bad-sample and empty-interval cases, events, every update/delete result code, unknown tags.
|
||||
- **Over-the-wire:** one `OpcUaServer.IntegrationTests` test drives HistoryUpdate Insert then HistoryRead
|
||||
through a real SDK client against a fake updater/data source.
|
||||
- **Live gate (docker-dev):** central pair on `Backend=SqlServer` against database `OtOpcUaHistorian` on
|
||||
the rig `sql` container; a historized rig tag records rows (Primary only — row count advances once per
|
||||
value); Client.CLI `historyread` raw + processed; `historyupdate` insert/replace/delete round-trip;
|
||||
stop the Primary ⇒ the Secondary takes over recording into the same table with no duplicate rows; a
|
||||
Bad sample is stored with its Bad status. Evidence recorded in `docs/plans/2026-10-09-pluggable-historian-live-gate.md`.
|
||||
|
||||
## 8. Docs and index
|
||||
|
||||
`docs/Historian.md` (backend selection, SQL schema, aggregate semantics, HistoryUpdate), `docs/Configuration.md`
|
||||
(`ServerHistorian:Backend` + `SqlServer` subsection), `docs/security.md` (HistoryUpdate role/permission),
|
||||
`docs/Client.CLI.md`, `CLAUDE.md` Historian section, and the scadaproj `sister-project-history/otopcua.md`
|
||||
entry (ScadaBridge impact: none unless it issues HistoryUpdate; HistoryRead semantics unchanged on the
|
||||
gateway backend apart from honest Bad quality on Bad samples).
|
||||
|
||||
## 9. Pre-execution review amendments (Fable, 2026-10-09)
|
||||
|
||||
These were verified against the code and override anything above that conflicts.
|
||||
|
||||
1. **Outbox recovery.** `FasterLogHistorizationOutbox`'s constructor (`RecoverState`) deserializes every record too, so the "unreadable ⇒ drop + count, never throw" rule applies there as well. An unreadable record is indexed under a synthetic id so FIFO truncation can pass it. If the v2 magic matches but structural validation fails, the legacy exact-length test runs before the entry is declared unreadable. `IHistorizationOutbox` gains `UnreadableCount` (default 0).
|
||||
2. **Validation** is as in §2 (all three gateway checks gated on `Backend`; consumer-gated backend validation). `HistorianUpdaterHolder.None` is always registered on driver nodes; the real updater only when `ServerHistorian:Enabled`.
|
||||
3. **Timestamps** are bound as `datetime2(7)` everywhere: `SqlDbType.DateTime2` scalar parameters and `SqlDataRecord` TVPs, never `DataTable`, which would round to 3.33 ms `datetime`.
|
||||
4. **Raw** `maxValuesPerNode == 0` means unbounded. **Events** fetch `maxEvents + 1` and return a non-null continuation point when truncated, as the interface requires.
|
||||
5. **Concurrency.** Tag creation and sample `MERGE`s use `WITH (HOLDLOCK)`, and `2627/2601` is treated as "already present". Both pair nodes provision on every deploy.
|
||||
6. **`Event`** has a nonclustered PK on `EventId` and is clustered on `(EventTimeUtc, EventId)`. The EventId hashes length-prefixed fields and includes `EquipmentPath`.
|
||||
7. **Processed reads** are capped at 100 000 buckets per request.
|
||||
8. **HistoryUpdate admission.** The `HistoryWrite` AccessLevel bit decides whether the SDK base admits a node to HistoryUpdate at all, so the updater must be wired before any node materialises. `OtOpcUaServerHostedService.StartAsync` assigns it (via the new `OtOpcUaSdkServer.SetHistorianDataUpdater`) before `SetSink`. Nodes materialised earlier keep their AccessLevel (no retro-update).
|
||||
9. **HistoryUpdate results.** Successful arms set `errors[i] = Good`. A `SourceTimestamp == DateTime.MinValue` is `BadInvalidTimestamp`.
|
||||
10. **Comms-loss IS historized, as an outage marker** (corrected after the T3 review). `DriverHostActor.FanCommsLossToDependencyMux` (#481) sends `null` + `BadNoCommunication` for every value ref to the mux, so the recorder receives it, and it cannot be told apart from a driver's own Bad publish. On a backend whose `SupportedKinds` include `Empty` (SQL Server), each disconnect stores one `Empty`/`BadNoCommunication` sample per historized tag. This is deliberate: it records the gap, so raw reads show the outage and aggregates and interpolation do not bridge it. On a numeric-only backend (Gateway) it is dropped as an unsupported kind and counted in `DroppedUnsupportedKind`. A driver's own Bad publish is handled the same way (`Empty` kind plus its Bad code, e.g. Modbus `0x80050000`).
|
||||
11. **Parallel safety.** `SqlServerTagCatalog` belongs to the skeleton task (T8), so the parallel SQL tasks share one.
|
||||
@@ -0,0 +1,133 @@
|
||||
# Pluggable historian — live gate (T15), 2026-10-09
|
||||
|
||||
**Verdict: PASSED.** All six legs passed on the docker-dev rig. The rig ran `feat/pluggable-historian` at
|
||||
`e2011a65` (plus the gate CLI fix), with the central pair on image `923f5f09e805`, rebuilt for this gate. The SQL Server backend
|
||||
pointed at the rig's `sql` container (database `OtOpcUaHistorian`).
|
||||
|
||||
The gate also found two defects:
|
||||
- A display gap in Client.CLI `historyupdate`. It is fixed in the same change.
|
||||
- An OpcUaClient driver comms-loss gap that predates this work. It is filed separately.
|
||||
|
||||
## Setup
|
||||
|
||||
- `CREATE DATABASE OtOpcUaHistorian` on `otopcua-dev-sql-1`. The schema `hist` was auto-created by the
|
||||
bootstrapper on first use: `SchemaVersion`, `Tag`, `Sample`, `Event`.
|
||||
- `docker-dev/docker-compose.yml` gained `ServerHistorian__Backend` (`OTOPCUA_HISTORIAN_BACKEND`, default
|
||||
`Gateway`) and `ServerHistorian__SqlServer__ConnectionString` (`OTOPCUA_HISTORIAN_SQL`).
|
||||
- Central-1 and central-2 were started with:
|
||||
|
||||
```
|
||||
OTOPCUA_HISTORIAN_ENABLED=true OTOPCUA_HISTORIAN_BACKEND=SqlServer OTOPCUA_CONTHIST_ENABLED=true \
|
||||
docker compose -f docker-dev/docker-compose.yml up -d central-1 central-2
|
||||
```
|
||||
|
||||
Both containers ran the new image, with `Backend=SqlServer`.
|
||||
- Cluster `MAIN` fixtures: opc-plc (`opcplc/plc/FastUInt1`, UInt32, about 1 Hz) and pymodbus `standard`
|
||||
(`pymodbus/plc/HR5`).
|
||||
- New Calculation String tag `Calc/eng/FastLabel` = `"v" + FastUInt1`. `FastUInt1`, `FastLabel` and later
|
||||
`HR5` were marked `isHistorized`. A UNS reference `a/l/e1/FastUInt1` was added. The configuration was
|
||||
deployed headlessly (`deploy.sh`), and the tags were provisioned:
|
||||
- `hist.Tag` 1 = FastLabel (DataType 9, String)
|
||||
- `hist.Tag` 2 = FastUInt1 (5, Int64)
|
||||
- `hist.Tag` 3 = HR5 (4)
|
||||
- Two data-plane API keys were minted for this gate, each with a `Cluster`-scoped grant:
|
||||
- `gate-historyupdate`: ReadOnly | HistoryUpdate (4239)
|
||||
- `gate-writeop`: ReadOnly | WriteOperate (159)
|
||||
|
||||
ACL flag off, which is the rig default.
|
||||
|
||||
## 1. Recording: Primary-only, typed, real status, one row per value
|
||||
|
||||
- Rows carry real kinds and statuses:
|
||||
- `ValueKind=2` Int64 `477`/`479`/…, status `0`
|
||||
- `ValueKind=4` String `v477`/`v479`/…, status `0`
|
||||
- startup `ValueKind=0` Empty, with `0x80320000` (BadWaitingForInitialData) and `0x80310000`
|
||||
- Primary-only:
|
||||
- Central-2 was the driver Primary, and its outbox held `hlog.log.0`.
|
||||
- Central-1's outbox directory was empty, and it logged "this node is not the recording Primary … suppressing
|
||||
value capture".
|
||||
- One row per published value, checked over 82 s:
|
||||
- 79 Good FastUInt1 rows and 79 Good FastLabel rows.
|
||||
- All 79 pair one-to-one: same `SourceTimeUtc`, `StrValue = 'v' + IntValue`.
|
||||
- 0 duplicate values.
|
||||
|
||||
## 2. HistoryRead through the raw and UNS NodeIds
|
||||
|
||||
These reads went to central-1 (`:4840`). Central-2's `:4841` returns "Server did not return the
|
||||
certificate used to create the secure channel" to the CLI; that rig port-mapping quirk is unrelated.
|
||||
|
||||
- Raw read 19:00:40–19:00:50 on `ns=2;s=opcplc/plc/FastUInt1` and on `ns=3;s=a/l/e1/FastUInt1`:
|
||||
- Identical 8 rows (493…502). Only the header line differs.
|
||||
- They equal the `hist.Sample` rows exactly.
|
||||
- The gaps (497, 501) come from the driver's 1 s sampling of a 1 s counter. They are in the stored rows and
|
||||
in the publish stream alike.
|
||||
- Average at 10 s, 19:00:40–19:01:10, through both NodeIds:
|
||||
- Both return `497.125 / 507.5 / 517.5`, Good.
|
||||
- These equal a direct SQL `AVG` per 10 s bucket (8, 10 and 10 samples).
|
||||
|
||||
## 3. HistoryUpdate
|
||||
|
||||
These calls went to central-1 with `-U apikey`. Each was checked by `historyread` and by direct SQL.
|
||||
|
||||
| Call | Result | Confirmed |
|
||||
|---|---|---|
|
||||
| insert 18:00:00 = 42 (`gate-historyupdate`) | `GoodEntryInserted` | row `42`, status 0 |
|
||||
| repeat insert = 43 | `BadEntryExists`, exit 1 | row still `42` |
|
||||
| replace 18:00:00 = 99 | `GoodEntryReplaced` | read and SQL show `99` |
|
||||
| replace 18:00:30 (no entry) | `BadNoEntryExists` | — |
|
||||
| delete-at-time 18:00:00 | `Good`, exit 0 | read returns 0 values; SQL row gone |
|
||||
| insert with `gate-writeop` (WriteOperate only) | Node result `BadUserAccessDenied`, exit 1 | no row |
|
||||
| insert, anonymous | Node result `BadUserAccessDenied`, exit 1 | no row |
|
||||
|
||||
**Gate finding (fixed):** the CLI printed only "(no result returned)" for a refused node. The refusal
|
||||
reason is in the node-level result, and the CLI never showed it. `HistoryUpdateCommand` now prints
|
||||
`Node result: <code>`, pinned by `Execute_NodeLevelDenial_PrintsNodeResult`. The rows above are from the
|
||||
fixed build.
|
||||
|
||||
## 4. Failover
|
||||
|
||||
- 19:03:08: `docker stop` central-2, the Primary.
|
||||
- 19:02:58: central-1 logged "this node is now the recording Primary … resuming value capture". It was
|
||||
promoted when central-2 began leaving. Its outbox appeared, and recording continued with no gap
|
||||
(633 → 644 …).
|
||||
- Central-2 was restarted and rejoined as standby:
|
||||
- Its `hlog.log.0` stayed frozen at 22528 bytes.
|
||||
- Central-1's grew (13824 → 15360).
|
||||
- Rate over the whole sequence was 1×. Per-30 s counts of FastUInt1 from 19:02:00 on: 30, 29, 29, 30, 30. There
|
||||
was no doubling during or after the failover.
|
||||
|
||||
## 5. Device loss: driver Bad sample and comms-loss outage marker
|
||||
|
||||
This leg ran on Modbus (`HR5`), as planned. Stopping `otopcua-pymodbus-standard` at 19:09:00 produced:
|
||||
|
||||
| SourceTimeUtc | ValueKind | Value | Status |
|
||||
|---|---|---|---|
|
||||
| 19:08:38.2408 | 2 | 5 | `0x00000000` |
|
||||
| 19:09:00.2649 | 0 (Empty) | NULL | `0x80050000` BadCommunicationError (driver's own Bad publish) |
|
||||
| 19:10:38.2327 | 0 (Empty) | NULL | `0x80310000` BadNoCommunication (#527 watchdog outage marker, ~98 s later) |
|
||||
| 19:12:38.4474 | 2 | 5 | `0x00000000` (fixture restarted 19:12:37; recovery re-publish) |
|
||||
|
||||
This confirms design §9.10 as amended: comms loss is historized as an outage marker. The plan's original
|
||||
leg-6 text predated that correction.
|
||||
|
||||
Reads over the outage:
|
||||
- Raw `historyread` shows Bad/Good/Bad/Bad/Good.
|
||||
- Average at 120 s gives `5 Uncertain` / `(null) Bad` / `5 Good`.
|
||||
|
||||
The outage is visible and is not bridged.
|
||||
|
||||
**Gate finding (pre-existing, filed separately):** stopping opc-plc first produced NO Bad sample. The
|
||||
OpcUaClient driver kept serving `FastUInt1 = 745`, status Good, for more than 3.5 minutes. It did not
|
||||
recover after opc-plc was restarted either. Its session loss never reaches the node status or driver
|
||||
health, so the #527 watchdog cannot fire. This is unrelated to the historian; Modbus behaves as designed.
|
||||
|
||||
## Restore
|
||||
|
||||
- The rig was restored afterwards:
|
||||
- central-1 and central-2 were recreated with the compose defaults (historian off).
|
||||
- The gate tags, script, UNS reference, API keys and ACL rows were removed.
|
||||
- The `HR5` and `FastUInt1` TagConfigs were reverted.
|
||||
- The configuration was redeployed.
|
||||
- `OtOpcUaHistorian` was dropped.
|
||||
- The opc-plc and pymodbus fixtures were stopped (normally down).
|
||||
- The compose keys stay, defaulting to `Gateway`.
|
||||
@@ -0,0 +1,645 @@
|
||||
# Pluggable Historian Backends Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development (recommended) or superpowers-extended-cc:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the historian backend pluggable (`ServerHistorian:Backend`), carry typed values + real status end to end, ship a SQL Server backend, and add the OPC UA HistoryUpdate service.
|
||||
|
||||
**Architecture:** A Runtime-level `IHistorianBackendProvider` builds the five historian seams (data source, value writer, alarm writer, provisioning, updater); the Host picks one by name. The existing recorder/outbox/alarm S&F pipelines are unchanged in structure and now carry a typed `HistorianValue` + `uint StatusCode`. SQL Server stores samples in a `(TagId, SourceTimeUtc)`-clustered table and pushes aggregates down to T-SQL.
|
||||
|
||||
**Tech Stack:** .NET 10, Akka.NET, OPC Foundation UA .NET Standard, Microsoft.Data.SqlClient (already pinned centrally), FASTER log (existing), xUnit v3 + Shouldly.
|
||||
|
||||
**Spec:** `docs/plans/2026-10-09-pluggable-historian-design.md` — the binding authority. Read it before any task.
|
||||
|
||||
## Global Constraints
|
||||
|
||||
- **No new NuGet packages, no version changes.** `Directory.Packages.props` is not edited by any task. Adding `<PackageReference Include="Microsoft.Data.SqlClient" />` (already pinned) to a new project is allowed.
|
||||
- Recording stays **Primary-only** (#537 capture gate) and **at-least-once** with the #538 per-tag quarantine and all-or-nothing ack. Do not restructure `DrainOnceAsync`.
|
||||
- In any Akka actor handler: capture `IActorRef self = Self` (and `Sender`) **before** the first `await`; never touch `Self`/`Context` after an `await …ConfigureAwait(false)` (#491).
|
||||
- Test fakes for async seams used by actors must complete on a **pool thread** (`await Task.Delay(1).ConfigureAwait(false)`), never synchronously and never via `Task.Yield()`.
|
||||
- Every OPC UA status code a new class emits is a **named `const uint`** in an internal `*StatusCodes` class, and that assembly must be visible to `tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/StatusCodeParityTests.cs` (it scans its bin dir — add a ProjectReference).
|
||||
- Authorization changes need an **ALLOW** test (a deny-only test passes against a build that never consults the gate).
|
||||
- A regression test must be shown to fail with the production change reverted (falsification) — record the observation in the task report.
|
||||
- The gateway backend's observable behaviour (bytes sent for Good numeric samples, provisioning skips, read paths) does not change.
|
||||
- Commit messages end with `Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe`. Subagents never push and never dispatch subagents.
|
||||
- macOS zsh: quote globs; `tea` needs `</dev/null`.
|
||||
|
||||
**User decisions (already made):**
|
||||
- Approach A (provider seam over the existing five interfaces).
|
||||
- SQLite dropped; SQL Server is the first new backend; QuestDB is a later spec.
|
||||
- "Update" = OPC UA HistoryUpdate exposed to clients (Insert/Replace/Update/DeleteRaw/DeleteAtTime).
|
||||
- Typed values (Double/Int64/Boolean/String/DateTime + Empty), not numeric-only.
|
||||
- Aggregates pushed down per database (T-SQL), launch subset per spec §4.
|
||||
- Flag-off authorization uses a NEW `HistoryUpdate` role, not `WriteOperate`.
|
||||
- Gateway config keys stay top-level; new backends get subsections.
|
||||
- Testing target: the docker-dev `sql` container (`otopcua-dev-sql-1`, `localhost,14330`).
|
||||
|
||||
---
|
||||
|
||||
## Execution model: waves, models, and parallel safety
|
||||
|
||||
### Model per role
|
||||
|
||||
| Role | Model | Why |
|
||||
|---|---|---|
|
||||
| Plan review (before execution) | **fable** | user-requested |
|
||||
| Implementer, mechanical/standard tasks (T1, T4, T5, T7, T8, T9, T11, T12, T13, T14) | **sonnet** | multi-file but fully specified |
|
||||
| Implementer, judgment-heavy tasks (T2 outbox crash boundary, T3 recorder actor, T6 node-manager HistoryUpdate + auth, T10 reads/aggregates) | **opus** | actor concurrency, auth semantics, SQL/Part 13 semantics |
|
||||
| Per-task reviewer | **sonnet** for sonnet tasks; **opus** for T2/T3/T6/T10 | scale to risk |
|
||||
| Fix rounds 4–5 escalation | one tier up (sonnet→opus, opus→fable) | per SDD skill |
|
||||
| Final whole-branch review | **fable** | cross-task integration risk |
|
||||
| Live gate (T15) | controller (this session) | needs rig + judgement; never delegated |
|
||||
|
||||
Always pass `model` explicitly on every Agent dispatch.
|
||||
|
||||
### Waves
|
||||
|
||||
```
|
||||
Wave 1 (serial): T1 contracts + compile ripple [sonnet]
|
||||
Wave 2 (parallel): T2 outbox v2 [opus] ‖ T3 recorder ‖ T4 gateway provider ‖ T6 server HistoryUpdate ‖ T7 CLI ‖ T8 SQL skeleton+schema
|
||||
Wave 3 (parallel): T5 registry+Host ‖ T9 SQL writes ‖ T10 SQL reads ‖ T11 SQL updater ‖ T13 over-the-wire test
|
||||
Wave 4 (serial): T12 SQL provider + Host registration
|
||||
Wave 5 (serial): T14 docs → final fable review → T15 live gate (controller)
|
||||
```
|
||||
|
||||
Dependencies: T2,T3,T4,T6,T7,T8 ← T1. T5 ← T4. T9,T10,T11 ← T8. T13 ← T6. T12 ← T5,T9,T10,T11. T14 ← T12,T13. T15 ← T14 + final review.
|
||||
|
||||
### Parallel-safety rules (binding)
|
||||
|
||||
1. **One git worktree per parallel task**, created by the controller under `.worktrees/ph-T<N>` from the integration branch `feat/pluggable-historian` at the wave's start commit. Run `git -C <wt> log -1` and confirm the base SHA in the dispatch (worktrees can start stale — reset first).
|
||||
2. **File ownership.** A task edits only the files its "Files" list names. The shared hot files have exactly one owner per wave:
|
||||
|
||||
| Shared file | Owner |
|
||||
|---|---|
|
||||
| `ZB.MOM.WW.OtOpcUa.slnx` | T8 (adds both SqlServer projects) |
|
||||
| `src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs` | T5 (wave 3), then T12 (wave 4) |
|
||||
| `.../Host/Configuration/ServerHistorianOptionsValidator.cs` | T5 (wave 3), then T12 (wave 4) |
|
||||
| `.../Runtime/Historian/ContinuousHistorizationMetrics.cs` | T5 |
|
||||
| `.../OpcUaServer/OtOpcUaSdkServer.cs`, `.../Host/OpcUa/OtOpcUaServerHostedService.cs` | T6 |
|
||||
| `tests/Server/.../Runtime.Tests/Historian/ContinuousHistorizationRecorderTests.cs` | T1 (ripple) then T3 |
|
||||
| `Core.Abstractions/Historian/IHistorizationOutbox.cs` | T1 |
|
||||
| `tests/Core/.../Core.Abstractions.Tests/*.csproj` | T8 |
|
||||
| `src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs` | T6 only |
|
||||
| `.../Runtime/Historian/ContinuousHistorizationRecorder.cs` | T1 (ripple) then T3 only |
|
||||
| `.../Driver.Historian.SqlServer/*.csproj` | T8 only (T9–T11 add `.cs` files only; csproj uses default globbing + `EmbeddedResource` for `Schema/*.sql` set by T8) |
|
||||
|
||||
A task that discovers it needs an un-owned shared file stops and reports `NEEDS_CONTEXT` rather than editing it.
|
||||
3. **New-file-only parallelism in the SQL project.** T9/T10/T11 each create their own files and their own test files; none edits another's. Shared SQL helpers live in T8's `SqlHistorianConnection.cs`/`SampleRowMapper.cs`; if a wave-3 task needs a helper change, it adds a private helper in its own file instead.
|
||||
4. **Merge order** at wave end: the controller merges worktree branches into `feat/pluggable-historian` in ascending task number, running `dotnet build ZB.MOM.WW.OtOpcUa.slnx` after each merge and the touched test projects after the last. A conflict ⇒ stop and resolve on the integration branch, never inside a worktree.
|
||||
5. **Integration-test DBs are per-test-run** (`OtOpcUaHistorian_T_<guid>`), so parallel T9/T10/T11 suites against the same `sql` container cannot collide.
|
||||
6. Subagents do not push, merge, or dispatch subagents. Only the controller merges.
|
||||
|
||||
### Common commands
|
||||
|
||||
```bash
|
||||
dotnet build ZB.MOM.WW.OtOpcUa.slnx
|
||||
dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests
|
||||
dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~Historian"
|
||||
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests
|
||||
export HISTORIAN_SQL_TEST_CONNECTION='Server=localhost,14330;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True;Encrypt=False'
|
||||
dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests
|
||||
```
|
||||
|
||||
The rig `sql` container must be up: `docker ps --filter name=otopcua-dev-sql-1`. If down: `cd docker-dev && docker compose up -d sql`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Typed historian contracts + compile ripple
|
||||
|
||||
**Goal:** Introduce `HistorianValue`, `HistorianValueKind`, `HistoryUpdateMode`, `IHistorianDataUpdater`, `IHistorianBackendProvider`, change `HistorizationValue`/`HistorizationOutboxEntry` to typed+status, and keep the whole solution compiling with **no behaviour change**.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - Add `long UnreadableCount => 0;` as a default interface member on `IHistorizationOutbox` (Core.Abstractions) — T2 overrides it, T5 binds the gauge. (#6)
|
||||
> - The record call sites to update are explicit: `grep -rn "HistorizationOutboxEntry(\|HistorizationValue(" src tests --include='*.cs'` (≈17 sites in 7 files incl. `GatewayLiveIntegrationTests.cs`, `HistorizationOutboxEntrySerializerTests.cs`, `GatewayHistorianValueWriterTests.cs`, `ContinuousHistorizationRecorderTests.cs`). Every one of them is in T1's Files. Do not stop at an unlisted one. (#3)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistorianValue.cs`
|
||||
- Create: `src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataUpdater.cs`
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/IHistorianBackendProvider.cs`
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianValueWriter.cs` (`HistorizationValue`)
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistorizationOutboxEntry.cs`
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorizationOutbox.cs` (default member `UnreadableCount`)
|
||||
- Modify (ripple only): `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationRecorder.cs` (lines ~548 entry construction, ~692 drain mapping), `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/GatewayHistorianValueWriter.cs`, `.../Recorder/HistorizationOutboxEntrySerializer.cs`, and every test that constructs these records (find with `grep -rn "HistorizationOutboxEntry(\|HistorizationValue(" src tests --include='*.cs'`).
|
||||
- Test: `tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/Historian/HistorianValueTests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] `HistorianValue.TryFromObject` implements exactly the spec §3 table; tests cover every row incl. `ulong` above `long.MaxValue` ⇒ Double, `DateTime` local ⇒ UTC ticks, `null` ⇒ Empty, `int[]` ⇒ false.
|
||||
- [ ] `TryGetDouble` true for Double/Int64/Boolean only; `ToObject` round-trips each kind (`DateTime` returns `DateTimeKind.Utc`).
|
||||
- [ ] Solution builds; all pre-existing historian tests pass unchanged in assertion content (only constructor call sites updated).
|
||||
- [ ] Ripple preserves behaviour: recorder still coerces numerics only (via `TryFromObject` + `TryGetDouble`, building `HistorianValue.FromDouble(numeric)`) and still stamps Good (`StatusCode = 0`); serializer still writes the legacy layout using `Value.Number` and quality `192`; gateway writer maps `Value.Number` and quality `192`.
|
||||
|
||||
**Verify:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests --filter "FullyQualifiedName~HistorianValue" && dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~Historiz"` → all green.
|
||||
|
||||
**Steps:**
|
||||
|
||||
- [ ] **Step 1: Write `HistorianValueTests` (failing — type does not exist).** Cases:
|
||||
|
||||
```csharp
|
||||
[Theory]
|
||||
[InlineData(1.5d, HistorianValueKind.Double)]
|
||||
[InlineData(1.5f, HistorianValueKind.Double)]
|
||||
[InlineData(7, HistorianValueKind.Int64)]
|
||||
[InlineData((short)7, HistorianValueKind.Int64)]
|
||||
[InlineData((byte)7, HistorianValueKind.Int64)]
|
||||
[InlineData(7u, HistorianValueKind.Int64)]
|
||||
[InlineData(7L, HistorianValueKind.Int64)]
|
||||
[InlineData(true, HistorianValueKind.Boolean)]
|
||||
[InlineData("x", HistorianValueKind.String)]
|
||||
public void Maps_clr_values_to_kinds(object input, HistorianValueKind expected)
|
||||
{
|
||||
HistorianValue.TryFromObject(input, out var v).ShouldBeTrue();
|
||||
v.Kind.ShouldBe(expected);
|
||||
}
|
||||
// plus: decimal 2.5m ⇒ Double 2.5; ulong.MaxValue ⇒ Double; (ulong)5 ⇒ Int64 5;
|
||||
// null ⇒ Empty (true); new[]{1,2} ⇒ false; new object() ⇒ false;
|
||||
// DateTime(2026,1,1,0,0,0,Local) ⇒ DateTime kind, Integer == ToUniversalTime().Ticks, ToObject() Kind Utc;
|
||||
// FromBoolean(true): Integer 1, Number 1, TryGetDouble ⇒ 1; FromString("a").TryGetDouble ⇒ false; Empty.ToObject() ⇒ null.
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run, confirm compile failure.**
|
||||
- [ ] **Step 3: Implement `HistorianValue.cs`:**
|
||||
|
||||
```csharp
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>The value kinds a historian backend can store. Persisted as a byte (SQL ValueKind, outbox v2) — never renumber.</summary>
|
||||
public enum HistorianValueKind : byte { Empty = 0, Double = 1, Int64 = 2, Boolean = 3, String = 4, DateTime = 5 }
|
||||
|
||||
/// <summary>A typed historized value. Number carries Double (and the double view of Int64/Boolean);
|
||||
/// Integer carries Int64, Boolean (1/0) and DateTime (UTC ticks); Text carries String.</summary>
|
||||
public readonly record struct HistorianValue(HistorianValueKind Kind, double Number, long Integer, string? Text)
|
||||
{
|
||||
public static HistorianValue Empty => default;
|
||||
public static HistorianValue FromDouble(double v) => new(HistorianValueKind.Double, v, 0, null);
|
||||
public static HistorianValue FromInt64(long v) => new(HistorianValueKind.Int64, v, v, null);
|
||||
public static HistorianValue FromBoolean(bool v) => new(HistorianValueKind.Boolean, v ? 1d : 0d, v ? 1 : 0, null);
|
||||
public static HistorianValue FromString(string v) => new(HistorianValueKind.String, 0d, 0, v ?? throw new ArgumentNullException(nameof(v)));
|
||||
public static HistorianValue FromDateTime(DateTime v) =>
|
||||
new(HistorianValueKind.DateTime, 0d, (v.Kind == DateTimeKind.Local ? v.ToUniversalTime() : DateTime.SpecifyKind(v, DateTimeKind.Utc)).Ticks, null);
|
||||
|
||||
public static bool TryFromObject(object? value, out HistorianValue result)
|
||||
{
|
||||
switch (value)
|
||||
{
|
||||
case null: result = Empty; return true;
|
||||
case double d: result = FromDouble(d); return true;
|
||||
case float f: result = FromDouble(f); return true;
|
||||
case decimal m: result = FromDouble((double)m); return true;
|
||||
case sbyte sb: result = FromInt64(sb); return true;
|
||||
case byte b: result = FromInt64(b); return true;
|
||||
case short s: result = FromInt64(s); return true;
|
||||
case ushort us: result = FromInt64(us); return true;
|
||||
case int i: result = FromInt64(i); return true;
|
||||
case uint ui: result = FromInt64(ui); return true;
|
||||
case long l: result = FromInt64(l); return true;
|
||||
case ulong ul: result = ul <= long.MaxValue ? FromInt64((long)ul) : FromDouble(ul); return true;
|
||||
case bool bo: result = FromBoolean(bo); return true;
|
||||
case string str: result = FromString(str); return true;
|
||||
case DateTime dt: result = FromDateTime(dt); return true;
|
||||
default: result = Empty; return false; // arrays, references, structures ⇒ not historizable
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryGetDouble(out double value)
|
||||
{
|
||||
switch (Kind)
|
||||
{
|
||||
case HistorianValueKind.Double:
|
||||
case HistorianValueKind.Int64:
|
||||
case HistorianValueKind.Boolean: value = Number; return true;
|
||||
default: value = 0d; return false;
|
||||
}
|
||||
}
|
||||
|
||||
public object? ToObject() => Kind switch
|
||||
{
|
||||
HistorianValueKind.Double => Number,
|
||||
HistorianValueKind.Int64 => Integer,
|
||||
HistorianValueKind.Boolean => Integer != 0,
|
||||
HistorianValueKind.String => Text,
|
||||
HistorianValueKind.DateTime => new DateTime(Integer, DateTimeKind.Utc),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: `IHistorianDataUpdater.cs`** — exactly the spec §4 "Updates" interface + `HistoryUpdateMode` enum, with XML docs naming the per-item status codes.
|
||||
- [ ] **Step 5: `IHistorianBackendProvider.cs`** in `ZB.MOM.WW.OtOpcUa.Runtime.Historian` — exactly spec §2.
|
||||
- [ ] **Step 6: Change the records:**
|
||||
|
||||
```csharp
|
||||
public readonly record struct HistorizationValue(DateTime? TimestampUtc, HistorianValue Value, uint StatusCode);
|
||||
public sealed record HistorizationOutboxEntry(Guid Id, string Tag, HistorianValue Value, uint StatusCode, DateTime TimestampUtc);
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Ripple, behaviour-preserving.** Recorder line ~527: replace `TryCoerceToDouble(msg.Value, out double numeric)` with `HistorianValue.TryFromObject(msg.Value, out var hv) && hv.TryGetDouble(out double numeric)` (same drop semantics — strings/null/DateTime still dropped as non-numeric); entry = `new HistorizationOutboxEntry(Guid.NewGuid(), historianName, HistorianValue.FromDouble(numeric), 0u /*Good*/, timestampUtc)`; delete `TryCoerceToDouble`. Drain line ~692: `new HistorizationValue(e.TimestampUtc, e.Value, e.StatusCode)`. Serializer: write `entry.Value.Number` and `(ushort)192`; read back as `FromDouble(value)`, status `0`. Gateway writer: `NumericValue = value.Value.Number, Quality = 192`. Keep `GoodQuality` const for now (T3 removes it). Update test call sites.
|
||||
- [ ] **Step 8: Build + run Verify.** Commit: `feat(historian): typed HistorianValue contracts, IHistorianDataUpdater, IHistorianBackendProvider (no behaviour change)`.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistorianValue.cs","src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianDataUpdater.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/IHistorianBackendProvider.cs","src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/IHistorianValueWriter.cs","src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/Historian/HistorizationOutboxEntry.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationRecorder.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/GatewayHistorianValueWriter.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/HistorizationOutboxEntrySerializer.cs"],"acceptanceCriteria":["TryFromObject matches spec table","solution builds, existing historian tests green","no behaviour change"],"modelTier":"standard","model":"sonnet","wave":1}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Outbox format v2 with legacy read
|
||||
|
||||
**Goal:** Serialize every `HistorianValueKind` + `uint StatusCode`; read pre-upgrade entries; drop (and count) unreadable entries instead of throwing.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - **Model: opus** (FIFO invariant on the crash boundary). (#1)
|
||||
> - `FasterLogHistorizationOutbox.RecoverState()` (ctor) ALSO deserializes every record. Use `TryDeserialize` in **both** `RecoverState` and `PeekBatchAsync`. In `RecoverState`, index an unreadable record in `_live/_index` under a synthetic `Guid.NewGuid()` with its address so FIFO truncation can pass it; in Peek skip it (and make sure an unreadable entry at the head is removable so it can't pin the head — e.g. Peek auto-removes leading unreadable entries). `UnreadableCount` (overrides T1's default member) increments in both. (#1)
|
||||
> - Required test: write a raw garbage record straight into the FasterLog directory (open `FasterLog` directly), **restart** the outbox, prove the ctor succeeds, the following good entry drains, and the head advances past the garbage. (#1)
|
||||
> - Legacy/v2 ambiguity: when the 4-byte magic matches but v2 structural validation fails, fall back to the legacy exact-length test before declaring unreadable. (#18)
|
||||
> - Drop the meter bullet: T5 binds the gauge via `IHistorizationOutbox.UnreadableCount`. (#6)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/HistorizationOutboxEntrySerializer.cs`
|
||||
- Modify: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/FasterLogHistorizationOutbox.cs` (only the deserialize call site in Peek: skip+count unreadable entries; expose `long UnreadableCount`)
|
||||
- Test: `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/Recorder/HistorizationOutboxEntrySerializerV2Tests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] v2 layout exactly: `"HV2\0"`(4) `ver=2`(1) `id`(16) `tagLen`(i32 LE) `tag`(UTF-8) `kind`(1) payload `status`(u32 LE) `ts`(i64 LE `ToBinary`). Payload: Empty none; Double f64; Int64/Boolean/DateTime i64; String i32 len + UTF-8.
|
||||
- [ ] Round-trip test per kind incl. empty string, a 70-KB string, `NaN`, `double.NegativeInfinity`, `long.MinValue`, status `0x80310000`.
|
||||
- [ ] **Pinned legacy bytes:** a hex literal produced by today's serializer (generate it once from the T1-era `Serialize` for `Id=00000000-0000-0000-0000-000000000001, Tag="t", Value=42.5, Quality=192, ts=2026-01-01T00:00:00Z` and paste the bytes into the test) deserializes to `Double 42.5`, status `0`.
|
||||
- [ ] `TryDeserialize` returns false (never throws) for: 3 random bytes, magic with ver=9, magic with truncated payload, legacy-looking bytes whose length disagrees with `38+tagLen`. `FasterLogHistorizationOutbox` skips such an entry on peek, advances past it (it must be removable so it can't pin the head — truncate through its id or log-address as the outbox already does for acked entries), and increments `UnreadableCount`; a test proves a corrupt entry between two good ones does not block the second.
|
||||
- [ ] Meter: `ContinuousHistorizationMetrics` gets an observable `otopcua.historian.outbox.unreadable` bound alongside the existing outbox gauges — **only if** `BindOutbox` lives in an un-owned file; otherwise report the counter on the outbox and note it for T5. (Owned file check: `ContinuousHistorizationMetrics.cs` is NOT owned by T2 — expose the property only.)
|
||||
|
||||
**Verify:** `dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests --filter "FullyQualifiedName~Outbox"` → green; falsification: with the legacy branch deleted the pinned-bytes test fails.
|
||||
|
||||
**Steps:** (1) capture the legacy hex from the current serializer via a throwaway test print, paste into the new test; (2) write failing v2 tests; (3) implement `Serialize` (v2 always) + `TryDeserialize(ReadOnlySpan<byte>, out HistorizationOutboxEntry?)` (v2 by magic, else legacy by exact length, else false) and keep a throwing `Deserialize` wrapper only if other callers need it; (4) wire skip+count in the outbox peek; (5) green; commit `feat(historian): outbox format v2 carries typed values + status; legacy entries still read`.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/HistorizationOutboxEntrySerializer.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/FasterLogHistorizationOutbox.cs"],"acceptanceCriteria":["v2 layout","pinned legacy bytes read","corrupt entry skipped and counted, never pins head"],"modelTier":"frontier","model":"opus","wave":2}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Recorder captures typed values with real status, filtered by backend kinds
|
||||
|
||||
**Goal:** The recorder records `HistorianValue` + `msg.StatusCode` and drops kinds the backend does not support.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - Add `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ContinuousHistorizationRecorderTests.cs` to Files — lines ~337 and ~1149 assert `DroppedNonNumeric` and must follow the rename. (#3)
|
||||
> - The recorder remarks ("carrying only (TagId, Value, TimestampUtc) and no quality") are stale: `DependencyMuxActor.cs:114,150` passes `item.StatusCode`. Rewrite them. (verified)
|
||||
> - Comms-loss watchdog `BadNoCommunication` stamps are node-only (`FanCommsLossToValueNodes` never reaches the mux) and are NOT historized — say so in the remarks; only a driver's own Bad publish (usually `null` value ⇒ `Empty`) is. (#14)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationRecorder.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs` **only** the recorder `Props` construction (~line 485–520) to pass `supportedKinds` — resolve `IHistorianBackendProvider` optionally; absent ⇒ `{Double, Int64, Boolean}` (today's behaviour).
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Historian/ContinuousHistorizationRecorderTypedValueTests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] New ctor/Props parameter `IReadOnlySet<HistorianValueKind> supportedKinds` (default = `{Double, Int64, Boolean}` for existing overloads so other tests compile unchanged).
|
||||
- [ ] Capture: `TryFromObject` false ⇒ drop + `_droppedUnsupportedKind++`; kind ∉ supportedKinds ⇒ same; otherwise entry carries the typed value and `msg.StatusCode`.
|
||||
- [ ] Status field in the recorder status record renamed `DroppedNonNumeric` → `DroppedUnsupportedKind`; any OTel instrument name stays unchanged (dashboards). `GoodQuality` const and its doc paragraph removed; class remarks updated (the mux DOES carry StatusCode — `VirtualTagActor.DependencyValueChanged`).
|
||||
- [ ] Tests (fake writer completes on a pool thread): a `Bad (0x80000000)` numeric sample is written with status `0x80000000`; a string sample is written when kinds include String and dropped+counted when not; `null` with status `BadNoCommunication` is written as Empty when kinds include Empty, dropped otherwise; an `int[]` is always dropped; existing #537 capture-gate and #538 quarantine tests still pass untouched.
|
||||
- [ ] Falsification recorded: reverting to the hard-coded Good makes the Bad-status test fail.
|
||||
|
||||
**Verify:** `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests --filter "FullyQualifiedName~ContinuousHistorization"` → green.
|
||||
|
||||
**Steps:** failing tests → implement → green → grep for `DroppedNonNumeric` across src/tests and update readers (AdminUI/metrics) **only within Runtime**; if a reader exists outside the owned files, report it for T5 instead of editing. Commit `fix(historian): record the real status code and typed values; backend-supported kinds gate capture`.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationRecorder.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ServiceCollectionExtensions.cs"],"acceptanceCriteria":["real status recorded","kind filter","#537/#538 unchanged","pool-thread fake"],"modelTier":"frontier","model":"opus","wave":2}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Gateway provider + reverse quality mapping
|
||||
|
||||
**Goal:** `GatewayHistorianProvider : IHistorianBackendProvider`; the gateway writer maps typed value + status to the gateway's numeric + DA quality.
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianProvider.cs`
|
||||
- Modify: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/GatewayQualityMapper.cs` (add `ToDaQuality(uint)`)
|
||||
- Modify: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/GatewayHistorianValueWriter.cs`
|
||||
- Test: `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayHistorianProviderTests.cs`, `.../Mapping/GatewayQualityMapperReverseTests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Provider: `Name = "Gateway"`, `SupportedKinds = {Double, Int64, Boolean}`, `CreateUpdater` returns `null`; the four `Create*` delegate to the existing `GatewayHistorian.Create…` (data source/alarm writer/provisioner) and build `GatewayHistorianValueWriter` exactly as `Program.cs` does today (`HistorianGatewayClientAdapter.Create(options, loggerFactory)`). Options come from the ctor (`ServerHistorianOptions`).
|
||||
- [ ] `ToDaQuality(Map(q)) == q` for every explicit byte in `Map`'s table; unknown status falls back by severity bits (`0x00…`⇒192, `0x40…`⇒64, `0x80…`⇒0).
|
||||
- [ ] Writer: a value with `TryGetDouble` false is skipped (debug log, no throw); Good Double 42.5 produces the same `HistorianLiveValue` (`NumericValue=42.5, Quality=192`) as before T1 — asserted in a test.
|
||||
|
||||
**Verify:** `dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests` → green.
|
||||
|
||||
**Steps:** tests first → implement → green → commit `feat(historian): GatewayHistorianProvider; status→DA quality reverse mapping`.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianProvider.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Mapping/GatewayQualityMapper.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/Recorder/GatewayHistorianValueWriter.cs"],"acceptanceCriteria":["provider shape","reverse mapping inverse","Good bytes unchanged"],"modelTier":"standard","model":"sonnet","wave":2}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Backend registry, config, validators, Host wiring
|
||||
|
||||
**Goal:** `ServerHistorian:Backend` selects a provider; Program.cs builds every historian seam through it.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - Gate **all three** gateway checks in `ServerHistorianOptionsValidator` (Endpoint, ApiKey, UseTls⇔scheme) on `Backend == Gateway`, and the same in `ServerHistorianOptions.Validate()` warnings. Tests: `Backend=SqlServer, Enabled=true, no Endpoint/ApiKey ⇒ passes`; control `Backend=Gateway, no ApiKey ⇒ fails` (unchanged). (#2)
|
||||
> - Unknown `Backend` ⇒ always fail (no I/O). Backend-specific option validation (SqlServer, added by T12) runs inside `ServerHistorianOptionsValidator` and only when consumed (`ServerHistorian:Enabled || AlarmHistorian:Enabled`) — the existing consumer-gated posture. `HistorianBackendRegistry.Resolve` must not validate. (#11)
|
||||
> - Register `HistorianUpdaterHolder.None` **unconditionally** in the driver block; register the real `new HistorianUpdaterHolder(provider.CreateUpdater(sp))` only when `ServerHistorian:Enabled` (HistoryUpdate gated like HistoryRead; last registration wins). (#11)
|
||||
> - Add `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationMetrics.cs` to Files: add observable gauge `otopcua.historian.outbox.unreadable` from `IHistorizationOutbox.UnreadableCount` in `BindOutbox`. (#6)
|
||||
> - Program.cs is top-level statements and not unit-testable: the "DI graph unchanged" AC is tested as `HistorianBackendRegistry.Resolve(gatewayOpts, cfg).CreateDataSource(sp) is GatewayHistorianDataSource` (and likewise for the writer/alarm writer/provisioner). Do not try to host-start Program. (#22)
|
||||
> - Spec wording fix: there are three `GatewayHistorian.Create…` call sites (Program.cs ~241/262/273) plus the inline writer (~313). (#16)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs` (add `Backend` string, default `"Gateway"`)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs` (lines ~215–318 only)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/HistorianBackendRegistryTests.cs`, extend `ServerHistorianOptionsValidatorTests`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Registry: `static IHistorianBackendProvider Resolve(ServerHistorianOptions o, IConfiguration cfg)`; knows `"Gateway"` now (T12 adds `"SqlServer"`); case-insensitive; unknown ⇒ `OptionsValidationException` naming the value and listing valid names. The registry also exposes `IReadOnlyCollection<string> KnownBackends` used by the validator.
|
||||
- [ ] Validator: unknown backend fails; `Endpoint` required only when backend is Gateway; existing tests unchanged in intent.
|
||||
- [ ] Program.cs: resolve the provider once; register it as a singleton `IHistorianBackendProvider`; replace the 4 `GatewayHistorian.Create…` lambdas and the inline `GatewayHistorianValueWriter` with `provider.Create…(sp)`; register `IHistorianDataUpdater` via `provider.CreateUpdater(sp)` **only when non-null** (`TryAddSingleton`-style factory is not possible for a null — register conditionally by calling it once at registration time is NOT allowed (needs sp); instead register a small `HistorianUpdaterHolder(IHistorianDataUpdater? Updater)` singleton, and have T6's hosted-service hook read `holder.Updater`). The alarm-only warning text names `serverHistorianOptions.Backend`.
|
||||
- [ ] Bind the outbox unreadable gauge from T2 in `ContinuousHistorizationMetrics.BindOutbox` if T2 reported it (this task owns that file this wave).
|
||||
- [ ] With `Backend` absent the Host's DI graph resolves the same concrete types as before (test: build the service collection with a Gateway config and assert `IHistorianDataSource` resolves to `GatewayHistorianDataSource`).
|
||||
|
||||
**Verify:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests --filter "FullyQualifiedName~Historian"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ContinuousHistorizationMetrics.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/ServerHistorianOptions.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/Program.cs"],"acceptanceCriteria":["unknown backend fails boot","Gateway default resolves identical types","updater holder registered"],"modelTier":"standard","model":"sonnet","wave":3}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 6: HistoryUpdate service in the node manager + authorization
|
||||
|
||||
**Goal:** Clients can HistoryUpdate historized nodes when the backend supplies an updater, gated by ACL (flag on) or the new `HistoryUpdate` role (flag off).
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - SDK pinned `OPCFoundation.NetStandard.Opc.Ua.* 1.5.378.106`. Verified: public `HistoryUpdate(OperationContext, Type detailsType, …)` runs under `Lock`, pre-seeds `errors[ii] = BadHistoryOperationUnsupported`, and admits a variable **only if `(AccessLevel & AccessLevels.HistoryWrite) != 0`**; it releases the lock and dispatches to protected `HistoryUpdateData` / `HistoryDeleteRawModified` / `HistoryDeleteAtTime(ServerSystemContext, IList<…Details>, IList<HistoryUpdateResult>, IList<ServiceResult>, List<NodeHandle>, IDictionary<NodeId,NodeState>)`. The HistoryWrite bit is therefore **dispatch-load-bearing**. (#4)
|
||||
> - Add `src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs` to Files: `SetHistorianDataUpdater(IHistorianDataUpdater?)` mirroring `SetHistorianDataSource` (line ~122). In `OtOpcUaServerHostedService.StartAsync` assign it **immediately after the `NodeManager is null` guard (~line 164), before `_deferredSink.SetSink` (~171)**; clear it in `StopAsync` next to the data-source clear (~373). Hosted service is registered before `AddAkka`, so this precedes every deploy apply. (#4, #5)
|
||||
> - On success each arm sets `errors[handle.Index] = ServiceResult.Good` (the base pre-seeds Unsupported; every HistoryRead success path here does the same, e.g. OtOpcUaNodeManager.cs ~5133). Per-node `HistoryUpdateResult.StatusCode` stays Good when per-item results are mixed; only a whole-node failure (deny / unsupported / not historized) sets the node code. (#13, #21)
|
||||
> - `UpdateDataDetails.UpdateValues[i].SourceTimestamp` is non-nullable: `== DateTime.MinValue` ⇒ `BadInvalidTimestamp` for that item. (#21)
|
||||
> - At least one test MUST drive the public `nm.HistoryUpdate(OperationContext, typeof(UpdateDataDetails), …)` entry point and assert the updater is called — it is the test that fails when the AccessLevel bit is missing. Tests that call the protected arm directly do not count toward the ALLOW criterion. (#4)
|
||||
> - AclEdit: just add `NodePermissions.HistoryUpdate` to the `PermissionBits` array (~161–166); the page renders enum names, there is no label table. (#23)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/OpcUaDataPlaneRoles.cs` (`HistoryUpdate = "HistoryUpdate"`)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyInterimRoleMapper.cs`
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs` (assign `HistorianDataUpdater` from an optional `HistorianUpdaterHolder` — define `HistorianUpdaterHolder` in `Runtime/Historian/HistorianUpdaterHolder.cs` in THIS task: `public sealed record HistorianUpdaterHolder(IHistorianDataUpdater? Updater) { public static readonly HistorianUpdaterHolder None = new((IHistorianDataUpdater?)null); }`, resolve with `GetService` ⇒ None when absent)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/AclEdit.razor` (add `NodePermissions.HistoryUpdate` to the list at ~line 163–166 with label "History update")
|
||||
- Modify: `src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Enums/NodePermissions.cs` comment only (remove "until the HistoryUpdate service surface is implemented"; still in no bundle)
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/History/HistoryUpdateTests.cs`, `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/OpcUa/ApiKeyInterimRoleMapperTests.cs` (extend)
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Before writing, read the pinned SDK's `CustomNodeManager2` and confirm the exact protected signatures for `HistoryUpdateData`, `HistoryDeleteRawModified`, `HistoryDeleteAtTime` (and how `HistoryUpdate` dispatches by details type); record them in the report. Use DeepWiki (`OPCFoundation/UA-.NETStandard`) if the source is not on disk.
|
||||
- [ ] `public IHistorianDataUpdater? HistorianDataUpdater { get; set; }` (volatile backing field, like `HistorianDataSource`).
|
||||
- [ ] `UpdateDataDetails`: `PerformUpdateType.Insert/Replace/Update` ⇒ `HistoryUpdateMode`; `Remove` ⇒ `BadHistoryOperationUnsupported`. Values: `DataValue` ⇒ `HistorizationValue(SourceTimestamp, TryFromObject(Value.Value) or BadTypeMismatch item, StatusCode.Code)`. Per-item results returned in `OperationResults` in order.
|
||||
- [ ] `DeleteRawModifiedDetails` with `IsDeleteModified=false` ⇒ `DeleteRawAsync`; `true` ⇒ unsupported. `DeleteAtTimeDetails` ⇒ `DeleteAtTimeAsync`.
|
||||
- [ ] Not historized / no updater ⇒ `BadHistoryOperationUnsupported`. Event/structure details ⇒ unsupported.
|
||||
- [ ] Authorization helper `AuthorizeHistoryUpdate` mirroring `AuthorizeHistoryRead` + the WriteOperate pattern at ~line 2024–2033: flag on ⇒ `_authorizer.Authorize(identity, OpcUaOperation.HistoryUpdate, nodeId, realm)`; flag off ⇒ `AuthorizeKeyUsable(...)` AND role `HistoryUpdate`. Deny ⇒ `BadUserAccessDenied`.
|
||||
- [ ] Batch limiter + deadline shared with HistoryRead (`WithHistoryReadBatchLimiter`, `CreateDeadline`).
|
||||
- [ ] `ComposeAccessLevel(writable, historized, historyWritable)` ORs `AccessLevels.HistoryWrite` when `historized && historyWritable`; `historyWritable = HistorianDataUpdater is not null` at materialise time. Raw + UNS mirrors both get it. (Setting the updater after nodes exist does not retro-update AccessLevel — document; the hosted service assigns it before the first deploy applies, verify the ordering and state it.)
|
||||
- [ ] `ApiKeyInterimRoleMapper` adds `OpcUaDataPlaneRoles.HistoryUpdate` for a Cluster-scoped grant with `NodePermissions.HistoryUpdate`; a WriteOperate-only grant does NOT confer it.
|
||||
- [ ] Tests (fake updater + fake authorizer, following the existing node-manager test harness for HistoryRead): **ALLOW flag-on with no roles + scoped grant ⇒ updater called and `GoodEntryInserted` returned**; flag-off with `HistoryUpdate` role ⇒ allowed; flag-off with only `WriteOperate` ⇒ `BadUserAccessDenied` and updater NOT called; expired key ⇒ denied; no updater ⇒ unsupported; per-item codes passed through in order; UNS NodeId resolves to the same tagname as raw.
|
||||
- [ ] Falsification recorded for the ALLOW test (break the flag-on branch ⇒ it fails).
|
||||
|
||||
**Verify:** `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests --filter "FullyQualifiedName~HistoryUpdate" && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests --filter "FullyQualifiedName~ApiKeyInterimRoleMapper"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaNodeManager.cs","src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/OtOpcUaSdkServer.cs","src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/Security/OpcUaDataPlaneRoles.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/ApiKeyInterimRoleMapper.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/OpcUa/OtOpcUaServerHostedService.cs","src/Server/ZB.MOM.WW.OtOpcUa.Runtime/Historian/HistorianUpdaterHolder.cs","src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Pages/Clusters/AclEdit.razor","src/Core/ZB.MOM.WW.OtOpcUa.Configuration/Enums/NodePermissions.cs"],"acceptanceCriteria":["SDK signatures verified","ALLOW test","WriteOperate does not confer HistoryUpdate","HistoryWrite access bit"],"modelTier":"frontier","model":"opus","wave":2}
|
||||
```
|
||||
|
||||
Note: T5 references `HistorianUpdaterHolder` — T5 runs in wave 3 after T6 merges, so the type exists.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Client.CLI `historyupdate` command
|
||||
|
||||
**Goal:** A CLI command to drive HistoryUpdate for manual + live-gate testing.
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/Commands/HistoryUpdateCommand.cs` (follow the existing `historyread` command's structure, options parsing, session/auth handling exactly)
|
||||
- Modify: `docs/Client.CLI.md` (new section)
|
||||
- Test: the Client.CLI test project if one exists for commands (`ls tests/Client`); follow its pattern for argument parsing tests.
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Options per spec §6; `--mode` values `insert|replace|update|delete-raw|delete-at-time`; `--value` parsed against `--type double|int64|bool|string|datetime` (default double); repeated `--time/--value` pairs; validation errors exit non-zero with a message.
|
||||
- [ ] Sends `UpdateDataDetails` / `DeleteRawModifiedDetails(IsDeleteModified=false)` / `DeleteAtTimeDetails` via `Session.HistoryUpdate`; prints `index time StatusCodeName` per item plus the overall service result.
|
||||
- [ ] Argument-parsing unit tests for each mode (no server needed).
|
||||
|
||||
**Verify:** `dotnet build src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI && dotnet test tests/Client --filter "FullyQualifiedName~HistoryUpdate"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/Commands/HistoryUpdateCommand.cs","docs/Client.CLI.md"],"acceptanceCriteria":["all five modes","arg parsing tests"],"modelTier":"standard","model":"sonnet","wave":2}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 8: SQL Server backend project skeleton, schema, bootstrapper, test fixture
|
||||
|
||||
**Goal:** New projects, embedded DDL, idempotent locked bootstrap, options + validation, shared connection/mapping helpers, and the env-gated integration fixture.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - **Move `SqlServerTagCatalog.cs` here from T9** (get-or-create + read-only lookup), used by T9/T10/T11/T12. Create path: `MERGE {s}.Tag WITH (HOLDLOCK) AS t …`; on `2627/2601` re-select the TagId. Integration test: two concurrent get-or-create on the same new name ⇒ both return the same id, one row. (#10, #12)
|
||||
> - Timestamps: helper `AddDateTime2(SqlCommand, name, DateTime)` (`SqlDbType.DateTime2`, scale 7) for every scalar param; TVPs are `IEnumerable<SqlDataRecord>` with `SqlMetaData(…, SqlDbType.DateTime2, 0, 7)` — **never a `DataTable`** (infers 3.33 ms `datetime`). Mapper tests use non-millisecond-aligned ticks (`…T00:00:00.1234567Z`). (#7)
|
||||
> - DDL: `CREATE SCHEMA` must be alone in its batch ⇒ `IF SCHEMA_ID(N'$(Schema)') IS NULL EXEC(N'CREATE SCHEMA [$(Schema)]')`; substitute `$(Schema)` **before** splitting on `GO`. (#17)
|
||||
> - `Event`: `EventId nvarchar(64) NOT NULL CONSTRAINT PK_Event PRIMARY KEY NONCLUSTERED`, plus `CREATE CLUSTERED INDEX CX_Event_Time ON {s}.Event (EventTimeUtc, EventId)` (drop the separate `IX_Event_Time`). Keep `IX_Event_Source_Time`. (#19)
|
||||
> - Private helper types introduced by wave-3 tasks inside this project must be prefixed with their task (`T9…`, `T10…`, `T11…`) to avoid merge collisions. (#12)
|
||||
> - Verified: `StatusCodes.*` names in `SqlHistorianStatusCodes` all exist in the pinned SDK, and `AggregateBits` Calculated=0x1 / Interpolated=0x2 with InfoType 0x400 are correct; keep the SDK-oracle unit test. (#26)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.csproj` (refs: Core.Abstractions, Core.AlarmHistorian, Runtime; `<PackageReference Include="Microsoft.Data.SqlClient" />`, `Microsoft.Extensions.Logging.Abstractions`; `<EmbeddedResource Include="Schema\*.sql" />`; `InternalsVisibleTo` both test projects)
|
||||
- Create: `.../SqlServerHistorianOptions.cs` (+ `Validate()` returning errors per spec §2), `.../SqlHistorianStatusCodes.cs`, `.../SqlHistorianConnection.cs` (open connection, schema-qualify helper `Q(string obj)` ⇒ `[{schema}].[{obj}]`, transient-error classifier: 1205, -2, 40613, 40197, 40501, 49918, 4060, 233, 10053, 10054, 10060, 11001, SqlException with `IsTransient`), `.../SampleRowMapper.cs` (HistorianValue ⇄ columns per spec §4 + `DataTable` builders for `SampleRow` and `TimeRow` TVPs), `.../SqlServerSchemaBootstrapper.cs`, `.../Schema/001_initial.sql`
|
||||
- Create: `docs/historian/sqlserver/001_initial.sql` (copy of the embedded script)
|
||||
- Create: `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/` (unit) and `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/` with `SqlHistorianFixture.cs`
|
||||
- Modify: `ZB.MOM.WW.OtOpcUa.slnx` (4 new projects), `tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests.csproj` (ProjectReference to the new src project so `StatusCodeParityTests` scans `SqlHistorianStatusCodes`)
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] `001_initial.sql` is the spec §4 DDL with `{s}` replaced by a `$(Schema)` token that the bootstrapper substitutes after validating the identifier regex; every statement guarded (`IF SCHEMA_ID(...) IS NULL`, `IF OBJECT_ID(...) IS NULL`, `IF TYPE_ID(...) IS NULL`) so a re-run is a no-op; ends with `INSERT SchemaVersion(1, SYSUTCDATETIME())` guarded by `IF NOT EXISTS`. Split on `^GO$` lines.
|
||||
- [ ] `SqlServerSchemaBootstrapper.EnsureAsync(ct)`: lazy, once per instance (a `SemaphoreSlim` + `_ready` flag; failure NOT latched), `sp_getapplock` exclusive session lock with a 30 s timeout around the apply; `AutoCreateSchema=false` ⇒ only checks `MAX(Version) >= 1` and throws `HistorianSchemaMissingException("…missing version 1…")`.
|
||||
- [ ] `SqlHistorianStatusCodes` contains every code the spec names (`Good, GoodNoData, GoodEntryInserted, GoodEntryReplaced, BadNoData, BadEntryExists, BadNoEntryExists, BadInvalidTimestamp, BadTypeMismatch, BadAggregateInvalidInputs, BadHistoryOperationUnsupported, UncertainDataSubNormal`) as named `const uint` with values copied from `Opc.Ua.StatusCodes` — the parity test passes and lists this class.
|
||||
- [ ] Historian info-bit constants (`InfoTypeDataValue = 0x00000400`, `HistorianCalculated = 0x1`, `HistorianInterpolated = 0x2`) in a separate `HistorianInfoBits` static class, with a unit test asserting `new Opc.Ua.StatusCode(Good).SetAggregateBits(AggregateBits.Calculated).Code == (Good | InfoTypeDataValue | HistorianCalculated)` (adjust to the SDK's actual helper — the test must compare against the SDK, not against this plan).
|
||||
- [ ] `SqlHistorianFixture` (xUnit `IAsyncLifetime`, collection fixture): reads `HISTORIAN_SQL_TEST_CONNECTION`; absent ⇒ `SkipReason` and tests `Assert.Skip(SkipReason)` (follow `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlPollServerFixture.cs`); present ⇒ `CREATE DATABASE OtOpcUaHistorian_T_<guid:N>`, exposes `ConnectionString` for it, `DROP DATABASE` (after `ALTER … SET SINGLE_USER WITH ROLLBACK IMMEDIATE`) on dispose.
|
||||
- [ ] Integration tests: bootstrap twice ⇒ second is a no-op and `SchemaVersion` has one row; two bootstrappers concurrently ⇒ both succeed, one row; `AutoCreateSchema=false` on an empty DB ⇒ `HistorianSchemaMissingException`; invalid schema name rejected by `Validate()` before any SQL runs (unit).
|
||||
- [ ] `SampleRowMapper` unit tests: every kind round-trips through the column triple.
|
||||
|
||||
**Verify:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests && dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests && dotnet test tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests --filter "FullyQualifiedName~StatusCodeParity"` → green (integration with the env var set; also run once WITHOUT it and confirm all skip).
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerTagCatalog.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/","tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.Tests/","tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests/","ZB.MOM.WW.OtOpcUa.slnx","docs/historian/sqlserver/001_initial.sql"],"acceptanceCriteria":["idempotent DDL","applock bootstrap","parity-guarded status consts","env-gated throwaway DB fixture"],"modelTier":"standard","model":"sonnet","wave":2}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 9: SQL Server writes — value writer, provisioning, alarm writer
|
||||
|
||||
**Goal:** Live values, tag provisioning and alarm events persist idempotently.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - `SqlServerTagCatalog` now comes from T8 — do NOT create it; use it. (#12)
|
||||
> - Insert-if-absent: catch `2627/2601` and treat as success (the row is durably present), so a concurrent writer never books a #538 failure. (#25)
|
||||
> - Provisioning `MERGE … WITH (HOLDLOCK)`; integration test with two concurrent provisioners of the same new tags ⇒ both report Ensured, no Failed. (#10)
|
||||
> - EventId: SHA-256 over **length-prefixed** UTF-8 fields `EquipmentPath, AlarmId, EventKind, TimestampUtc.Ticks, User, Comment, Message` (null ⇒ length −1). Unit test that `"a|b"+"c"` and `"a"+"b|c"` hash differently. (#20)
|
||||
> - All timestamp round-trips use non-ms-aligned ticks and assert tick equality. (#7)
|
||||
|
||||
**Files (create only):**
|
||||
- `.../SqlServerHistorianValueWriter.cs`, `.../SqlServerTagProvisioner.cs`, `.../SqlServerAlarmHistorianWriter.cs`
|
||||
- Tests: `tests/.../SqlServer.IntegrationTests/WriteTests.cs`, `ProvisioningTests.cs`, `AlarmWriterTests.cs`; unit `tests/.../SqlServer.Tests/AlarmEventIdTests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Every public entry point calls `bootstrapper.EnsureAsync` first.
|
||||
- [ ] Value writer: TVP insert-if-absent (`INSERT INTO Sample(...) SELECT @tagId, r.* , SYSUTCDATETIME() FROM @rows r WHERE NOT EXISTS (SELECT 1 FROM Sample s WITH (UPDLOCK, HOLDLOCK) WHERE s.TagId=@tagId AND s.SourceTimeUtc=r.SourceTimeUtc)`); within-batch duplicate timestamps collapse to the first (de-dup in C# before building the TVP — TVP PK would reject them); null timestamp ⇒ `DateTime.UtcNow`; returns false (never throws) on SQL errors; logs category only.
|
||||
- [ ] Provisioning: one `MERGE` per batch upserting Name/DataType/EU/Description; returns `(Requested, Ensured, Skipped:0, Failed)`; never throws.
|
||||
- [ ] Alarm writer: EventId = lowercase hex SHA-256 of `AlarmId|EventKind|TimestampUtc.Ticks|User|Comment|Message` (UTF-8, `|` separator, null ⇒ empty); insert-if-absent; per-event outcome per spec (transient classifier from T8).
|
||||
- [ ] Integration: each kind written then read back by direct SQL equals input; writing the same batch twice leaves row count unchanged and returns true; a 10 000-value batch completes < 5 s on the rig; provisioning upsert changes DataType on re-run; same alarm event twice ⇒ one row, both `Ack`; an over-long `AlarmTypeName` (> 100) ⇒ `PermanentFail`.
|
||||
|
||||
**Verify:** `dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests --filter "FullyQualifiedName~Write|FullyQualifiedName~Provisioning|FullyQualifiedName~AlarmWriter"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianValueWriter.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerTagProvisioner.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerAlarmHistorianWriter.cs"],"acceptanceCriteria":["idempotent redelivery","never throws","deterministic EventId"],"modelTier":"standard","model":"sonnet","wave":3}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 10: SQL Server reads — raw, at-time, events, processed (push-down)
|
||||
|
||||
**Goal:** `SqlServerHistorianDataSource : IHistorianDataSource` per spec §4 "Reads".
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - `maxValuesPerNode == 0` is the node manager's unbounded sentinel ⇒ `TOP(2147483647)`; test it. (#8)
|
||||
> - Events: `TOP(@maxEvents + 1)` (maxEvents 0 ⇒ unbounded); when the extra row is present drop it and return a non-null opaque ContinuationPoint (e.g. UTF-8 of `EventTimeUtc.Ticks|EventId`) — the interface says MUST. Test `maxEvents=2` over 3 rows. (#9)
|
||||
> - Processed: cap the bucket count at 100 000; above it throw `ArgumentOutOfRangeException` and document that the node manager surfaces it as a per-node Bad. Check what the node manager's processed path does with a thrown non-aggregate exception and match it. (#24)
|
||||
> - Tag lookup uses T8's `SqlServerTagCatalog` read-only lookup. All timestamp tests non-ms-aligned. (#7, #12)
|
||||
|
||||
**Files (create only):** `.../SqlServerHistorianDataSource.cs`, `.../SqlAggregateQueries.cs` (one method per aggregate family returning the SQL text), tests `tests/.../SqlServer.IntegrationTests/RawReadTests.cs`, `AtTimeReadTests.cs`, `EventReadTests.cs`, `ProcessedReadTests.cs`.
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Tag lookup is read-only (`SELECT TagId FROM Tag WHERE Name=@n`, cached on hit only) — reads never create tags.
|
||||
- [ ] Raw forward/reverse/limit exactly per spec; `DataValueSnapshot` built with `Value.ToObject()`, `StatusCode`, source + server timestamps (inspect `DataValueSnapshot`'s ctor first).
|
||||
- [ ] AtTime: single statement with `TimeRow` TVP and two `OUTER APPLY (SELECT TOP 1 …)`; interpolation/stepping/no-prior/bad-neighbour rules per spec; status built from `SqlHistorianStatusCodes` + `HistorianInfoBits`.
|
||||
- [ ] Processed: generates every interval in `[start,end)` (empty ⇒ `BadNoData`), the aggregate table in spec §4, Calculated bit on every result, Uncertain when any Bad present, `BadAggregateInvalidInputs` for numeric aggregates on String/DateTime tags, `HistoryAggregateNotSupportedException` for the rest. `interval <= 0` ⇒ single interval `[start,end)`.
|
||||
- [ ] Reverse processed (`end < start`) ⇒ swap and return descending (document).
|
||||
- [ ] `GetHealthSnapshot()` tracks totals/consecutive failures/last error (the bootstrap error surfaces here).
|
||||
- [ ] Integration fixture data: tag `A` with samples at t0+0s=10, +10s=20, +20s=Bad(30), +30s=40, +40s=50 (Double); tag `S` strings; `B` booleans. Hand-computed expectations for each supported aggregate over 20 s intervals from t0 — write them as literal expected values in the test (e.g. interval [t0,t0+20s): Average 15, Min 10, Max 20, Total 30, Count 2, Start 10, End 20, Delta 10, Range 10, PercentGood 100; interval [t0+20s,t0+40s): usable {40}, Average 40 with Uncertain status, PercentBad 50).
|
||||
- [ ] Falsification: drop the Bad-sample filter ⇒ the Uncertain interval test fails.
|
||||
|
||||
**Verify:** `dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests --filter "FullyQualifiedName~Read"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataSource.cs","src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlAggregateQueries.cs"],"acceptanceCriteria":["raw fwd/rev","AtTime interp/step","aggregate table with hand-computed literals","unsupported aggregates throw"],"modelTier":"frontier","model":"opus","wave":3}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 11: SQL Server updater
|
||||
|
||||
**Goal:** `SqlServerHistorianDataUpdater : IHistorianDataUpdater` per spec §4 "Updates".
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - Use T8's `SqlServerTagCatalog` for create-on-Insert/Update (remove the "private MERGE" fallback). Sample `MERGE … WITH (HOLDLOCK)`. (#10, #12)
|
||||
> - "Null timestamp" means `DateTime.MinValue` (the node manager maps it to `BadInvalidTimestamp` before calling, but the updater must still treat `MinValue` the same defensively). (#21)
|
||||
> - Non-ms-aligned ticks in every Replace/DeleteAtTime test. (#7)
|
||||
|
||||
**Files (create only):** `.../SqlServerHistorianDataUpdater.cs`; tests `tests/.../SqlServer.IntegrationTests/UpdateTests.cs`.
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] One transaction per call; `MERGE Sample AS t USING @rows AS r ON t.TagId=@tagId AND t.SourceTimeUtc=r.SourceTimeUtc` with `OUTPUT $action, r.SourceTimeUtc`; Insert uses `WHEN NOT MATCHED THEN INSERT` only (matched ⇒ BadEntryExists), Replace `WHEN MATCHED THEN UPDATE` only (not matched ⇒ BadNoEntryExists), Update both. Results mapped back to the caller's original order (including within-request duplicate timestamps: second occurrence ⇒ `BadEntryExists` for Insert, last-wins for Replace/Update — document).
|
||||
- [ ] Null timestamp ⇒ `BadInvalidTimestamp` for that item, others proceed.
|
||||
- [ ] Delete raw `[start,end)` ⇒ `Good` with rows deleted, `BadNoData` when 0; DeleteAtTime per time.
|
||||
- [ ] Unknown tag: Insert/Update create it via T8's `SqlServerTagCatalog`; Replace/Delete ⇒ BadNoEntryExists/BadNoData.
|
||||
- [ ] Integration test per row of the spec's result table + the duplicate/null cases; replaced value is visible via direct SQL with the new status.
|
||||
|
||||
**Verify:** `dotnet test tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer.IntegrationTests --filter "FullyQualifiedName~Update"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianDataUpdater.cs"],"acceptanceCriteria":["Part 11 result codes","order preserved","single transaction"],"modelTier":"standard","model":"sonnet","wave":3}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 12: SQL Server provider + Host registration
|
||||
|
||||
**Goal:** `Backend=SqlServer` works end to end in the Host.
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - SqlServer option validation lives in `ServerHistorianOptionsValidator`, consumer-gated (see T5 amendment) — add it there (this task owns that file in wave 4), not in `Resolve`. (#11)
|
||||
> - Add `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/ServerHistorianOptionsValidator.cs` to Files. One shared `SqlServerSchemaBootstrapper` + `SqlServerTagCatalog` per provider. (#11, #12)
|
||||
|
||||
**Files:**
|
||||
- Create: `src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs` (one shared bootstrapper + tag catalog instance across the five seams; `SupportedKinds` = all six kinds)
|
||||
- Modify: `src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs` (add `"SqlServer"`, binding `ServerHistorian:SqlServer` to `SqlServerHistorianOptions` and failing on its `Validate()` errors), `src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj` (ProjectReference), `Program.cs` only if T5 left a seam that needs it
|
||||
- Test: `tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests/Configuration/HistorianBackendRegistryTests.cs` (extend), SqlServer.Tests `SqlServerHistorianProviderTests.cs`
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] `Backend=SqlServer` with no ConnectionString ⇒ host-start validation failure naming `ServerHistorian:SqlServer:ConnectionString`.
|
||||
- [ ] DI: `IHistorianDataSource` ⇒ `SqlServerHistorianDataSource`, `IHistorianValueWriter` ⇒ SqlServer writer, `HistorianUpdaterHolder.Updater` non-null, recorder `supportedKinds` contains String and Empty.
|
||||
- [ ] The Endpoint requirement does not fire for SqlServer.
|
||||
|
||||
**Verify:** `dotnet build ZB.MOM.WW.OtOpcUa.slnx && dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.Host.Tests --filter "FullyQualifiedName~Historian"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer/SqlServerHistorianProvider.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/HistorianBackendRegistry.cs","src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj"],"acceptanceCriteria":["validation names the key","DI resolves SqlServer types","updater present"],"modelTier":"standard","model":"sonnet","wave":4}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Over-the-wire HistoryUpdate test
|
||||
|
||||
**Goal:** Prove HistoryUpdate works through a real SDK client session (the encoding/dispatch the node-manager unit tests mock around).
|
||||
|
||||
> **Review amendments (Fable, 2026-10-09) — binding, override anything below that conflicts:**
|
||||
> - Install the fake updater (`Nm.HistorianDataUpdater = fake`, same slot the rig uses for `Nm.Authorizer`, cf. `AclKeyExpiryEnforcementTests.cs:141-215`) **before** materialising the historized node. Add a control test that materialises first, then installs the updater, and proves HistoryUpdate returns `BadHistoryOperationUnsupported` (documents the no-retro-update rule). Assert the per-node `StatusCode` as well as `OperationResults`. (#4, #13, #15)
|
||||
|
||||
**Files (create only):** `tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/HistoryUpdateOverTheWireTests.cs` (use the existing integration harness that `ConditionMethodEventTests` uses; inject an in-memory fake `IHistorianDataUpdater` + `IHistorianDataSource` sharing one dictionary).
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Client HistoryUpdate Insert of 2 values on a historized node ⇒ `GoodEntryInserted` ×2; HistoryRead raw returns them; Insert again ⇒ `BadEntryExists`; DeleteAtTime one ⇒ read returns one.
|
||||
- [ ] A session without the grant ⇒ `BadUserAccessDenied`, fake not called.
|
||||
- [ ] The node's AccessLevel read over the wire includes `HistoryWrite`.
|
||||
|
||||
**Verify:** `dotnet test tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests --filter "FullyQualifiedName~HistoryUpdateOverTheWire"` → green.
|
||||
|
||||
```json:metadata
|
||||
{"files":["tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/HistoryUpdateOverTheWireTests.cs"],"acceptanceCriteria":["insert/read/duplicate/delete over the wire","deny case","HistoryWrite bit visible"],"modelTier":"standard","model":"sonnet","wave":3}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Documentation
|
||||
|
||||
**Goal:** Docs match the shipped behaviour.
|
||||
|
||||
**Files:** `docs/Historian.md`, `docs/Configuration.md`, `docs/security.md`, `CLAUDE.md` (Historian / HistoryRead section: add backend selection, SQL Server, HistoryUpdate, the real-status fix, and the gateway C-002 quality limitation; keep it a short subsection), `docs/drivers/README` or index if historians are listed there.
|
||||
|
||||
**Acceptance Criteria:**
|
||||
- [ ] Every config key in spec §2 documented with default and validation rule.
|
||||
- [ ] Aggregate table + "simple per-sample definitions, not full Part 13" caveat.
|
||||
- [ ] HistoryUpdate: operations, result codes, authorization (flag on/off), AccessLevel bit, unsupported cases.
|
||||
- [ ] Docker-dev recipe for a SQL historian (`OtOpcUaHistorian` on the `sql` container) and the integration-test env var.
|
||||
- [ ] No claim the docs make is untested — cross-check each against the merged code.
|
||||
|
||||
**Verify:** controller reads the diff; `grep -n "Backend" docs/Configuration.md`.
|
||||
|
||||
```json:metadata
|
||||
{"files":["docs/Historian.md","docs/Configuration.md","docs/security.md","CLAUDE.md"],"acceptanceCriteria":["keys documented","aggregate caveat","HistoryUpdate auth documented"],"modelTier":"standard","model":"sonnet","wave":5}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Final review (fable), then Task 15: Live gate on docker-dev (controller)
|
||||
|
||||
**Final review:** dispatch a **fable** reviewer over `git diff master...feat/pluggable-historian` with the spec; it must specifically check #491 (`Self` after await), #537/#538 unchanged, every new status literal is a named const, ALLOW tests exist, no `Directory.Packages.props` edit, gateway Good-bytes unchanged. One fix dispatch (opus) + scoped re-review (sonnet), then adjudicate.
|
||||
|
||||
**Task 15 — Live gate (controller only; not delegated).** **USER-ORDERED GATE — NON-SKIPPABLE.** Close only after each item below is re-validated with captured output.
|
||||
|
||||
1. `CREATE DATABASE OtOpcUaHistorian` on `otopcua-dev-sql-1`; set on central-1 and central-2 (compose env): `ServerHistorian__Enabled=true`, `ServerHistorian__Backend=SqlServer`, `ServerHistorian__SqlServer__ConnectionString=Server=sql,1433;Database=OtOpcUaHistorian;User Id=sa;Password=OtOpcUa!Dev123;TrustServerCertificate=True`, `ContinuousHistorization__Enabled=true` + `OutboxPath`. Rebuild images (the rig image goes stale), `up -d`.
|
||||
2. Mark a rig tag historized (one numeric, one string) and deploy. Evidence: `hist.Sample` row count advances by exactly one per published value (Primary-only), status column real.
|
||||
3. `Client.CLI historyread` raw + processed (Average, 10 s) on the raw NodeId AND its UNS NodeId — same rows.
|
||||
4. `Client.CLI historyupdate` insert (new timestamp) ⇒ `GoodEntryInserted`; repeat ⇒ `BadEntryExists`; replace ⇒ `GoodEntryReplaced`; delete-at-time ⇒ `Good`; each confirmed by `historyread` and by direct SQL. Use an API key whose principal has a Cluster-scoped `HistoryUpdate` grant; a key with only WriteOperate ⇒ `BadUserAccessDenied`.
|
||||
5. Stop the Primary container; the Secondary resumes recording into the same table; no duplicate `(TagId, SourceTimeUtc)` (guaranteed by PK — also check count growth rate stays 1×); restart.
|
||||
6. Stop the driver's device (or blackhole) **while the driver is still publishing** (before #527 trips Reconnecting) ⇒ a row with `ValueKind = 0 (Empty)` and the driver's Bad code (Modbus: `BadCommunicationError 0x80050000`). Watchdog `BadNoCommunication` stamps are node-only and are NOT historized — do not expect them.
|
||||
7. Record evidence in `docs/plans/2026-10-09-pluggable-historian-live-gate.md`; restore the rig config (Backend off) unless the user wants it kept.
|
||||
|
||||
```json:metadata
|
||||
{"files":["docs/plans/2026-10-09-pluggable-historian-live-gate.md"],"acceptanceCriteria":["Primary-only one row per value","historyread raw+processed via raw and UNS NodeIds","historyupdate insert/duplicate/replace/delete codes","WriteOperate-only key denied","failover continues recording","Bad sample stored Bad"],"userGate":true,"tags":["user-gate"],"verifyCommand":"Client.CLI historyread/historyupdate against opc.tcp://localhost:4840 + sqlcmd on otopcua-dev-sql-1","modelTier":"frontier","model":"controller","wave":5}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Finish
|
||||
|
||||
Merge `feat/pluggable-historian` → `master` only with the user's go-ahead; push only with approval. Then update the scadaproj `sister-project-history/otopcua.md` (on scadaproj `main`, pushed) and the index row only if the summary changed (driver/backends list). File Gitea follow-ups: QuestDB backend spec; retention/purge; Part 11 modified values; time-weighted aggregates.
|
||||
@@ -0,0 +1,455 @@
|
||||
# QuestDB historian backend — design
|
||||
|
||||
**Status:** approved section by section on 2026-10-10. Gitea #666. **As-built pass (T8, 2026-10-11):** every section below was re-checked against
|
||||
the merged code; where the two disagreed the text was corrected inline and marked **(as built)**. User documentation: `docs/Historian.md` §"QuestDB backend".
|
||||
**Builds on:** `docs/plans/2026-10-09-pluggable-historian-design.md`, which defines the provider seam, typed values, the
|
||||
HistoryUpdate service and the SQL Server backend, all merged to master as `8ec7a83c`.
|
||||
|
||||
## 0. Decisions (user, 2026-10-10)
|
||||
|
||||
| Topic | Decision |
|
||||
|---|---|
|
||||
| HistoryUpdate deletes | **Soft delete.** Rows carry a `deleted` flag. A delete writes a tombstone at the same `(tag, ts)`. Every read and aggregate excludes tombstones. All five HistoryUpdate operations are supported. |
|
||||
| Write path | **Postgres wire protocol only, via `Npgsql`.** It is already centrally pinned at 10.0.3. No ILP, no new NuGet package. |
|
||||
| Retention | **Included.** `RetentionDays` is applied as table TTL. QuestDB drops expired day partitions itself. |
|
||||
| Approach | **B: shared dialect-free core plus per-backend SQL.** The pure logic moves out of the SQL Server backend into `Driver.Historian.Sql.Common`. One conformance suite runs against both backends. |
|
||||
| Timestamps | **Microseconds.** Postgres-wire timestamps are microsecond resolution, and real source timestamps are at best milliseconds. Two samples less than 1 µs apart collapse to one, and the first wins — within a batch, and also against an already-stored row (the existence check runs on the rounded time). This is documented. |
|
||||
|
||||
Out of scope:
|
||||
- event HistoryUpdate;
|
||||
- Part 11 modified values (#668);
|
||||
- time-weighted aggregates (#671);
|
||||
- ILP ingest;
|
||||
- QuestDB authentication beyond the connection string;
|
||||
- any change to the Gateway backend.
|
||||
|
||||
## 1. Components
|
||||
|
||||
### 1.1 `ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common` (new, no I/O)
|
||||
|
||||
This is extracted from `Driver.Historian.SqlServer` with **no behaviour change**:
|
||||
|
||||
- **`StoredSample`.** It holds the time, the value, the stored status. It classifies a sample as
|
||||
usable, good, bad or uncertain. It applies the quality-contract clamp: a NULL value carrying a non-Bad status is
|
||||
served as `BadNoData`. It also defines the info-bit masks (`CodeBitsMask`, `SeverityBadBit`, `SeverityMask`).
|
||||
- **`AtTimeEvaluator`.** This is the former `SqlServerHistorianDataSource.AtTimeValue`. It covers interpolation and
|
||||
stepping, the outage rule (previous neighbour Bad ⇒ null plus that code; previous usable and next Bad ⇒ hold the
|
||||
previous value as `UncertainDataSubNormal`), stepping forward after the last sample, `T > now` ⇒ Uncertain, and
|
||||
masking `& ~0x3` before the info bits are set.
|
||||
- **`BucketRow` + `AggregateEvaluator`.** This is the former `Evaluate`. Each backend's SQL produces the **same
|
||||
per-bucket statistics**: total count, usable/good/bad/uncertain counts, numeric sum/min/max, first and last value
|
||||
and time, the time of the min and of the max, and so on. The shared evaluator maps a `BucketRow` to the OPC UA
|
||||
value and status for each of the 13 supported aggregates. Unsupported aggregates throw
|
||||
`HistoryAggregateNotSupportedException`.
|
||||
- **`BucketPlan`.** It handles window normalisation (a reversed window is swapped and returned descending, as built
|
||||
today), interval 0 ⇒ one bucket, the 100,000-bucket cap, and tick arithmetic. It exposes the bucket start and width
|
||||
in **ticks** and in **microseconds**.
|
||||
- **`HistorianColumnMapping`.** It maps a `HistorianValueKind` to the stored data-type code (the former
|
||||
`SqlHistorianDataTypes.FromKind`). It also maps a `HistorianValue` to its value columns (num/int/str) and back.
|
||||
- **`HistoryUpdateResults`.** These are the per-item result rules for Insert, Replace, Update, DeleteAtTime and
|
||||
DeleteRaw given "did a live row exist". They include the NaN/Inf rule (rejected `BadInvalidArgument` on update,
|
||||
stored as `BadOutOfRange` Empty on the live write) and the rule rejecting an Empty value with a non-Bad status.
|
||||
- **Shared constants.** The named status consts that both backends emit (`BadOutOfRange` and others) move into one
|
||||
parity-visible `*StatusCodes` class here. Backend-specific codes stay with their backend.
|
||||
|
||||
`Driver.Historian.SqlServer` then keeps only its T-SQL, the connection, the bootstrapper, the tag catalog and the
|
||||
seam classes, all calling the shared core.
|
||||
|
||||
### 1.2 `ZB.MOM.WW.OtOpcUa.Driver.Historian.QuestDb` (new)
|
||||
|
||||
| Class | Responsibility |
|
||||
|---|---|
|
||||
| `QuestDbHistorianOptions` | Binds `ServerHistorian:QuestDb` (§4) and validates it. |
|
||||
| `QuestDbConnection` | Npgsql `NpgsqlDataSource` (pooled), command timeouts, transient-error classification, UTC timestamp reads. |
|
||||
| `QuestDbSchemaBootstrapper` | Idempotent `CREATE TABLE IF NOT EXISTS` for the three tables, then applies TTL. It is concurrency-safe across two nodes starting together, and caches a failure for 5 s. |
|
||||
| `QuestDbHistorianValueWriter` | `IHistorianValueWriter` for live samples (§2.2). |
|
||||
| `QuestDbTagCatalog` | **(as built)** The only "is this tag known" authority (R7): a `hist_tag` row, positive results cached. Reads never create a tag; the live writer and HistoryUpdate Insert/Update do (Replace and the deletes do not). Re-inserting a known tag is skipped, because the dedup upsert would erase the unit and description the provisioner wrote. |
|
||||
| `QuestDbWalVisibility` | **(as built)** The read-your-writes wait (§2.3), used by every write path. |
|
||||
| `QuestDbTimeClusters` | **(as built)** Groups request times into ranges (gap > 1 min starts a new one) so existence reads are one range query per cluster. |
|
||||
| `QuestDbTables`, `QuestDbAggregateQueries` | **(as built)** DDL / TTL helpers, and the push-down aggregate SQL (§3). |
|
||||
| `QuestDbTagProvisioner` | `IHistorianProvisioning`: upserts `hist_tag` rows in chunks; **(as built)** a request without a unit or description keeps the stored one. |
|
||||
| `QuestDbAlarmHistorianWriter` | `IAlarmHistorianWriter`: **(as built)** one INSERT per event (not batched), with the deterministic SHA-256 `event_id`. Outcomes: Ack; `RetryPlease` for transient and unclassified errors; `PermanentFail` only for a pre-1970 time or a server message naming an inconvertible / out-of-range / too-long / null value. |
|
||||
| `QuestDbHistorianDataSource` | `IHistorianDataSource` (§3), with QuestDB SQL for raw, at-time, events and bucket statistics. |
|
||||
| `QuestDbHistorianDataUpdater` | `IHistorianDataUpdater` (§2.3), the soft-delete updater. |
|
||||
| `QuestDbHistorianProvider` | `IHistorianBackendProvider`: `Name = "QuestDb"`, `SupportedKinds` = all six (FrozenSet), one shared connection and bootstrapper. |
|
||||
|
||||
Host changes:
|
||||
- `HistorianBackendRegistry` gains `"QuestDb"`.
|
||||
- `ServerHistorianOptionsValidator` validates the QuestDb section, gated on consumers, the same way it gates
|
||||
SqlServer.
|
||||
- `Host.csproj` gets a ProjectReference.
|
||||
- **(as built)** `IHistorianDataSource` gains the default member `TimestampResolutionTicks` (1; QuestDB 10), and the node manager's Raw paging steps off a drained timestamp by it (§3).
|
||||
- **(as built)** The provider builds its shared pieces (connection, bootstrapper, catalog, WAL wait) on the first `Create*` call, not in the constructor, so a provider nobody consumes never fails on its options.
|
||||
|
||||
## 2. Schema and writes
|
||||
|
||||
### 2.1 Tables
|
||||
|
||||
QuestDB has no schemas, so the tables carry a configurable `TablePrefix` (default `hist_`, `[A-Za-z0-9_]+`). All
|
||||
three are **WAL tables with `DEDUP UPSERT KEYS`**, so re-sending a row with the same key replaces it.
|
||||
|
||||
**T0-confirmed** against `questdb/questdb:10.0.1` (`docs/plans/2026-10-10-questdb-spike.md` §DDL, Q3, Q5, Q10). The
|
||||
`TTL n DAYS` clause goes **between `PARTITION BY` and `WAL`**. Placed after the dedup clause it fails with `unexpected
|
||||
token [TTL]`. It is emitted only when `RetentionDays` is set, as an inlined integer, because it cannot be a parameter.
|
||||
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS hist_sample (
|
||||
ts TIMESTAMP, tag SYMBOL, kind BYTE, num_value DOUBLE, int_value LONG, str_value VARCHAR,
|
||||
status LONG, deleted BOOLEAN, server_ts TIMESTAMP
|
||||
) TIMESTAMP(ts) PARTITION BY DAY [TTL n DAYS] WAL DEDUP UPSERT KEYS(ts, tag);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hist_event (
|
||||
ts TIMESTAMP, event_id VARCHAR, source SYMBOL, alarm_id VARCHAR, alarm_name VARCHAR, alarm_type_name VARCHAR,
|
||||
severity INT, event_kind VARCHAR, message VARCHAR, user_name VARCHAR, comment VARCHAR, received_utc TIMESTAMP
|
||||
) TIMESTAMP(ts) PARTITION BY DAY [TTL n DAYS] WAL DEDUP UPSERT KEYS(ts, event_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS hist_tag (
|
||||
ts TIMESTAMP, name SYMBOL, data_type BYTE, eng_unit VARCHAR, description VARCHAR, updated_utc TIMESTAMP
|
||||
) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name); -- ts is always 1970-01-01, which makes this an upsert by name
|
||||
```
|
||||
|
||||
`hist_event` maps `hist.Event` one-for-one:
|
||||
- `EventTimeUtc` → `ts`, `EventId` → `event_id`, `SourceName` → `source`;
|
||||
- `AlarmId`/`AlarmName`/`AlarmTypeName` → `alarm_id`/`alarm_name`/`alarm_type_name`;
|
||||
- `Severity` → `severity`, `EventKind` → `event_kind`, `Message` → `message`;
|
||||
- `UserName` → `user_name`, `Comment` → `comment`, `ReceivedTimeUtc` → `received_utc`.
|
||||
|
||||
`event_id VARCHAR` works as a dedup key (T0 Q3), so re-sending an event replaces it. The S4 fallback (no dedup on
|
||||
`hist_event`) is not used.
|
||||
|
||||
Column details (**T0-confirmed**):
|
||||
- `status` holds the uint status code **unsigned** in a LONG (0..4294967295, never negative). So "severity Bad" is
|
||||
`status >= 2147483648`, not `< 0` as in SQL Server's signed `int`.
|
||||
- `int_value` also holds Booleans as 0/1, matching SQL Server.
|
||||
- `kind` is `HistorianValueKind` in a `BYTE`.
|
||||
- Read-back CLR types:
|
||||
|
||||
| Column type | Npgsql type name | CLR type / reader call |
|
||||
|---|---|---|
|
||||
| `BYTE` | `smallint` | `Int16` (`GetInt16`) |
|
||||
| `SYMBOL`, `VARCHAR` | `character varying` | `String` |
|
||||
| `LONG` | `bigint` | `Int64` |
|
||||
| `INT` | `integer` | `Int32` |
|
||||
| `BOOLEAN` | `boolean` | `Boolean` |
|
||||
| `DOUBLE` | `double precision` | `Double` |
|
||||
| `TIMESTAMP` | `timestamp without time zone` | `DateTime`, **`Kind=Unspecified`**; the reader applies `SpecifyKind(…, Utc)` |
|
||||
|
||||
- **NULL semantics:**
|
||||
- DOUBLE NaN and ±Inf read back as NULL.
|
||||
- `long.MinValue` (LONG) and `int.MinValue` (INT) are QuestDB's NULL sentinels.
|
||||
- `BYTE` and `BOOLEAN` have **no NULL**: unset or DBNull reads `0` / `false`. So `deleted` defaults to `false`, and a
|
||||
LEFT-JOINed "no row" yields `kind = 0`. Absence must be detected on a NULL `ts`, not on `kind`.
|
||||
- **Ruling RC2 (as built):** an Int64 sample of exactly `long.MinValue` stores `int_value` as NULL. The QuestDB read
|
||||
path repairs it before the shared `FromColumns`: `kind = Int64 ∧ int_value IS NULL ∧ num_value = -2^63` ⇒
|
||||
`long.MinValue`. That double is exact. Without the repair the sample would be served as 0.
|
||||
- A designated timestamp **before 1970-01-01 cannot be inserted** (`designated timestamp before 1970-01-01 is not
|
||||
allowed`). See §2.2 (RC1).
|
||||
- Tag names: SYMBOL and VARCHAR take 400 characters (non-ASCII included). SYMBOL grows past its capacity hint
|
||||
(16,384) with no error, and was tested with 30,000 distinct tags. No `CAPACITY` clause is needed.
|
||||
|
||||
### 2.2 Live writes (recorder) — insert only if absent, the same as SQL Server
|
||||
|
||||
1. Group the batch by tag and remove duplicates within the batch. **The first value wins.** A timestamp is rounded
|
||||
to microseconds before this step, so microsecond collisions also collapse.
|
||||
2. For each tag, fetch the timestamps that already exist. **(as built)** The batch's times are grouped into clusters
|
||||
(`QuestDbTimeClusters`: a gap over 1 minute starts a new range; the live writer also caps a range at 1,000 times), with one range query per cluster, so one
|
||||
far-away outlier does not widen a single query over every stored row in between. **Tombstones are included**, so a re-sent
|
||||
outbox batch can never resurrect a deleted row or overwrite a HistoryUpdate change. T0-confirmed form (Q11): `SELECT ts
|
||||
FROM {p}sample WHERE tag = $1 AND ts >= $2 AND ts <= $3`, with no `NOT deleted`, followed by a client-side set test. The tag is
|
||||
registered in `QuestDbTagCatalog` first (a write creates an unknown tag).
|
||||
3. Insert only the new rows with parameterised multi-row `INSERT … VALUES (…),(…)`, **at most 1,000 rows per
|
||||
statement** (T0-confirmed Q2). QuestDB reads the Bind parameter count as a signed int16, so the hard limit is
|
||||
32,767 parameters. **(as built)** The live writer binds 8 parameters per row (`deleted` is the literal `false`; 4,095 rows
|
||||
would fit), the updater 9 (3,640 rows). Within one statement a duplicate key keeps the **last** row, which is
|
||||
why step 1's "first wins" is done client-side.
|
||||
4. Stored-value rules are shared with SQL Server:
|
||||
- NaN/±Inf, or any Empty with a non-Bad status, is stored Empty with `BadOutOfRange` or `BadNoData`.
|
||||
- Comms-loss outage markers are stored as given.
|
||||
- A tag name over 400 characters is a per-tag failure.
|
||||
- **Ruling RC1 (T0 Q13):** a sample timestamped before 1970-01-01 cannot be stored, because QuestDB rejects
|
||||
the whole INSERT. The writer drops such samples before the INSERT, logs one Warning per batch and counts them
|
||||
(`DroppedPre1970`); the call still succeeds for the rest. A retry could
|
||||
never succeed, and sending the sample would push the tag's whole group into the #538 quarantine.
|
||||
5. Any SQL or transport failure makes the write return `false`, so the recorder retries and the #538 per-tag
|
||||
quarantine still applies. It never throws.
|
||||
|
||||
The recorder is Primary-only (#537), so step 2 does not race another live writer. The recorder waits for WAL
|
||||
visibility, bounded by `VisibilityTimeout` (plan R5), using the §2.3 wait.
|
||||
|
||||
Binding (T0-confirmed Q1/Q6/Q13):
|
||||
- Every timestamp parameter is `NpgsqlDbType.TimestampTz` with `DateTimeKind.Utc`, after `Round` (**as built:** Unspecified is
|
||||
taken as UTC, Local is converted, then ticks are truncated to a multiple of 10). `BindTimestamp(…, clampToEpoch: true)` is used
|
||||
only for write binds of a designated timestamp; read bounds are never clamped (a pre-1970 bound is valid, and clamping it would
|
||||
move the bound across the epoch row). `Timestamp` + `Utc`
|
||||
throws in Npgsql. Npgsql truncates sub-µs ticks **toward 2000-01-01**, which rounds *up* before 2000. A raw
|
||||
`DateTime.MinValue`/`MaxValue` is sent as ±infinity, and a `[Min, Max)` range then silently matches nothing. R6's
|
||||
`BindTimestamp` (round; `MinValue` → `+10` ticks; `MaxValue` → `−9` ticks) fixes all three.
|
||||
- The other parameter types:
|
||||
|
||||
| Column | `NpgsqlDbType` |
|
||||
|---|---|
|
||||
| `tag`/`str_value`/other strings | `Varchar` |
|
||||
| `kind`/`data_type` | `Smallint` |
|
||||
| `num_value` | `Double` |
|
||||
| `int_value`/`status` | `Bigint` |
|
||||
| `deleted` | `Boolean` |
|
||||
| `severity` | `Integer` |
|
||||
|
||||
NULL is `DBNull.Value` with the column's type.
|
||||
- Every QuestDB server error carries **SQLSTATE `00000`**. `QuestDbConnection`'s classification therefore treats a
|
||||
`PostgresException` as a server-side rejection, judged by `MessageText`. Any other `NpgsqlException`, `IOException`,
|
||||
`SocketException` or timeout is transport (transient).
|
||||
|
||||
### 2.3 HistoryUpdate (soft delete)
|
||||
|
||||
Every operation is an `INSERT`, and the dedup upsert performs the replace. QuestDB `UPDATE` and `DELETE` are never
|
||||
issued. "Exists" means a row that is present **and not deleted**.
|
||||
|
||||
| Operation | Effect | Result |
|
||||
|---|---|---|
|
||||
| Insert | Write the row when none exists; **a tombstone counts as absent, so Insert over a tombstone returns `GoodEntryInserted`** (an explicit client insert may re-populate a deleted time; only the live writer's redelivery is barred, §2.2) | `GoodEntryInserted` / `BadEntryExists` |
|
||||
| Replace | Overwrite the row when one exists | `GoodEntryReplaced` / `BadNoEntryExists` |
|
||||
| Update | Write the row either way | `GoodEntryReplaced` or `GoodEntryInserted` |
|
||||
| DeleteAtTime | Write a tombstone (`deleted = true`, value NULL) where a row exists | `Good` / `BadNoEntryExists` |
|
||||
| DeleteRaw | Write a tombstone for every live row in the range | the same codes SQL Server returns for the same cases |
|
||||
|
||||
Further rules:
|
||||
- The per-item rules come from `HistoryUpdateResults`, shared with SQL Server, so the codes are identical.
|
||||
- Item order is preserved, with exactly one code per item.
|
||||
- **Read-your-writes.** After its inserts, the updater polls `wal_tables()` until the table's applied transaction
|
||||
reaches the sequencer transaction it produced, up to `VisibilityTimeout`. On a timeout the write is kept, the codes
|
||||
are reported as normal, and a Warning is logged.
|
||||
- **The visibility wait (`QuestDbWalVisibility`, T0-confirmed Q4)** is used by the writer and the updater (R5):
|
||||
1. After the INSERT(s) return: `SELECT sequencerTxn FROM wal_tables() WHERE name = $1` (`$1` = the full table name,
|
||||
`Varchar`) → `target`. Each autocommitted INSERT is exactly one sequencer transaction.
|
||||
2. Poll `SELECT writerTxn, suspended FROM wal_tables() WHERE name = $1` about every 10 ms until `writerTxn >=
|
||||
target`. Stop early, with a Warning, if `suspended` is true. Stop at `VisibilityTimeout`.
|
||||
- Measured: median 0 ms, p95 6 ms, max 13 ms. Up to 7/20 inserts were not visible to an immediate re-read, so the
|
||||
wait is required. A concurrent writer can only raise `target`. The key-poll fallback (S11) is not used.
|
||||
- **(as built)** Existence checks for item times (Insert/Replace/Update/DeleteAtTime) are **not** `IN` lists: they use the T0 Q11 range
|
||||
form, one query per `QuestDbTimeClusters` cluster, `SELECT ts FROM {p}sample WHERE tag = $1 AND ts >= $2 AND ts <= $3 AND NOT
|
||||
deleted`, and the set test is client-side. The `IN ($2, …)` form this section originally proposed is not used.
|
||||
- **Unknown tag (as built):** Insert and Update register the tag in `QuestDbTagCatalog`; Replace and the deletes do not (Replace ⇒
|
||||
`BadNoEntryExists`, DeleteAtTime ⇒ `BadNoEntryExists`, DeleteRaw ⇒ `BadNoData`).
|
||||
- **Tombstone shape (as built):** `kind = Empty (0)`, value columns NULL, `status = BadNoData`, `deleted = true`.
|
||||
- **DeleteRaw (T0-confirmed), page size 1,000:**
|
||||
1. `SELECT ts FROM {p}sample WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts LIMIT 1000`.
|
||||
2. Write one 1,000-row tombstone INSERT.
|
||||
3. Run the visibility wait.
|
||||
4. Continue from `last ts + 10 ticks`, until a page comes back empty.
|
||||
The wait between pages is required, or the next page re-reads rows just tombstoned. Measured: 2,500 rows, 3 pages,
|
||||
103 ms.
|
||||
- **Ruling RC1 (T0 Q13):** an Insert, Replace or Update item timestamped before 1970-01-01 returns
|
||||
`BadOutOfRange`, because QuestDB cannot store it (nothing is sent; a null or `MinValue` timestamp is still
|
||||
`BadInvalidTimestamp`, and **as built** a pre-1970 item that is also NaN reports `BadOutOfRange`, not `BadInvalidArgument`,
|
||||
because the pre-1970 filter runs first). DeleteAtTime there returns `BadNoEntryExists`; DeleteRaw finds nothing. A DeleteRaw
|
||||
*window* may start before 1970. Pre-1970 conformance seeds are skipped for QuestDB.
|
||||
- **Concurrency (as built).** The updater serializes calls **per tag, in-process only** (64 striped async locks, held from the
|
||||
existence read through the visibility wait), so two concurrent Insert/Replace/Delete batches on one tag cannot both act on a stale
|
||||
read. It does **not** lock against the live writer or a second node. The existence checks read committed data, so a live recorder
|
||||
sample at the *same* timestamp still in flight in the WAL can race the updater, and **two nodes writing the same new key
|
||||
concurrently can produce last-write-wins** (QuestDB has no `UPDLOCK`; the Primary gate on the recorder makes it rare). Updates
|
||||
normally target past timestamps, so this residual is accepted and documented.
|
||||
|
||||
### 2.4 Retention
|
||||
|
||||
When `RetentionDays` is set, the bootstrapper applies TTL to `hist_sample` and `hist_event`, and re-applies it when
|
||||
the configured value changes. QuestDB then drops expired day partitions, tombstones included. When unset, data is
|
||||
kept forever. `hist_tag` never expires.
|
||||
|
||||
T0-confirmed (Q5):
|
||||
- **Create:** `… PARTITION BY DAY TTL n DAYS WAL …` (§2.1).
|
||||
- **Change:** `ALTER TABLE {p}sample SET TTL n DAYS`. **Remove** (`RetentionDays` unset while the table has a TTL):
|
||||
`ALTER TABLE … SET TTL 0 DAYS`.
|
||||
- `n` cannot be a `$` parameter (`invalid syntax, should be <number> <unit> but was $1`). It is the validated integer,
|
||||
formatted with `InvariantCulture`.
|
||||
- TTL is not allowed on the `PARTITION BY YEAR` `hist_tag` table (`TTL value must be an integer multiple of the
|
||||
partition size`).
|
||||
- **TTL is enforced only on write commits (as built, N3).** QuestDB looks for expired partitions when a write commits to the table,
|
||||
so a quiet table never drops anything; old partitions linger until the next write.
|
||||
- **Reading the current TTL:** `SELECT ttlValue, ttlUnit FROM tables() WHERE table_name = $1`. QuestDB **normalises
|
||||
the unit**: 7 days reads `1 WEEK`, 14 days `2 WEEK`, 30 days `30 DAY`, none `0 HOUR`. The bootstrapper compares in
|
||||
hours (`HOUR`=1, `DAY`=24, `WEEK`=168; any other unit counts as different, RC3) and issues an `ALTER` only on a
|
||||
difference. A large `RetentionDays` that QuestDB normalises to MONTH/YEAR therefore re-`ALTER`s on every boot (harmless).
|
||||
`AutoCreateSchema=false` touches no TTL.
|
||||
- `ALTER … SET TTL` is applied through the WAL, so `tables()` shows the old value for 25–100 ms. The bootstrapper does
|
||||
not re-read to verify.
|
||||
- Expiry is checked against the **wall clock**: the image default is `cairo.ttl.use.wall.clock=true`. A future-dated
|
||||
sample does not expire current data (T0 Q5c).
|
||||
- **Table listing:** `SELECT table_name FROM tables() WHERE starts_with(table_name, $1)`. Never `LIKE 'hist_%'`,
|
||||
because `_` is a wildcard.
|
||||
- **Concurrent bootstrap:** `CREATE TABLE IF NOT EXISTS` from 4 connections × 20 rounds produced no errors. The S11
|
||||
defence stays: catch a `PostgresException` whose `MessageText` contains `already exists`, re-check `tables()`, and
|
||||
rethrow if the table is absent. Never key on SQLSTATE (always `00000`).
|
||||
|
||||
## 3. Reads
|
||||
|
||||
Every query filters `NOT deleted`. Every value goes through `StoredSample`, so the quality clamp is the same as SQL
|
||||
Server's.
|
||||
|
||||
- **Raw.** A range on `ts`, `ORDER BY ts [DESC]`, `LIMIT @top` (`maxValuesPerNode == 0` ⇒ unbounded), at most `top` samples, `ContinuationPoint: null`; the node manager pages by re-reading from the last timestamp (plan R8). The dedup key forbids same-instant ties. Bounds follow the existing data-source contract: forward `[start, end)`, reverse `(end, start]`
|
||||
descending, and **(as built)** `start == end` reads the sample exactly at that (rounded) time. **(as built)** The data source reports
|
||||
`TimestampResolutionTicks = 10`, and the node manager steps off the last drained timestamp by that amount (forward `T + 10`, reverse
|
||||
`T − 10`): a 1-tick step is truncated back to `T`, so a one-value-per-page forward read would loop forever. The rule is *not* "trim
|
||||
everything ≤ the last emitted", because the same paging serves the Gateway backend, where same-timestamp ties are legal.
|
||||
- **T0-confirmed form (review fix):** `SELECT ts, kind, num_value, int_value, str_value, status FROM {p}sample WHERE
|
||||
tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts [DESC] LIMIT $4`.
|
||||
- `$4` is a **bound** parameter, `NpgsqlDbType.Integer` (Int32) or `Bigint` (Int64); both work on 10.0.1, ASC and
|
||||
DESC. Use `Integer`, since `top` is an `int`.
|
||||
- `int.MaxValue` (the R8 "unbounded" value) and `long.MaxValue` both return every row. `LIMIT 0` returns no rows,
|
||||
but R8 never sends 0.
|
||||
- The inlined-literal fallback is not needed.
|
||||
- **AtTime (T0-confirmed Q9).** Find the previous-or-equal neighbour and the next neighbour of each requested time
|
||||
with `LIMIT 1` seeks, `UNION ALL`-ed, **at most 50 request times (100 branches) per statement**. The exact QuestDB
|
||||
10.0.1 limit is **126 `UNION ALL` branches per statement (63 request times)**: 126 work, and 127 fail with `SQL
|
||||
model is too complex to evaluate`. So 50 times leaves a 26-branch margin. Per time `k`, with `$1` = tag and `$j` = the
|
||||
rounded time:
|
||||
```sql
|
||||
SELECT * FROM (SELECT k AS req, 0 AS dir, ts, kind, num_value, int_value, str_value, status FROM {p}sample
|
||||
WHERE tag = $1 AND ts <= $j AND NOT deleted ORDER BY ts DESC LIMIT 1)
|
||||
UNION ALL
|
||||
SELECT * FROM (SELECT k AS req, 1 AS dir, ts, kind, num_value, int_value, str_value, status FROM {p}sample
|
||||
WHERE tag = $1 AND ts > $j AND NOT deleted ORDER BY ts ASC LIMIT 1)
|
||||
```
|
||||
`k` and `dir` are generated integer literals. Measured: 102 times in about 16–20 ms over 100k rows per tag, with no
|
||||
mismatches. `ASOF JOIN` was faster (1 ms), but it yields only the previous neighbour, so it was not chosen.
|
||||
`AtTimeEvaluator` then decides the result. **(as built, M3)** Each time is evaluated at its *rounded* value but the result is stamped
|
||||
with the time the client *requested* (UTC-normalised, not truncated), as on SQL Server, so answers match requests. Stored data is µs-only,
|
||||
so a sub-µs request adjacent to a sample is an exact hit.
|
||||
- **Events.** Filtered by source and `ts` range `[start, end)`, ordered `(ts, event_id)` ascending (reversed windows are swapped, as built
|
||||
today). **(as built)** With a bound, `maxEvents + 1` rows are fetched; the extra row is dropped and becomes the continuation point.
|
||||
- **Processed.** **One query per node** (T0-confirmed for every family; the S11 two-statement fallback is not needed).
|
||||
- Bucket index (T0-confirmed): `(cast(ts AS LONG) * 10 - $1) / $2`.
|
||||
- `cast(ts AS LONG)` is µs since 1970.
|
||||
- `$1` = the rounded window start in **ticks since the Unix epoch** (`start.Ticks − DateTime.UnixEpoch.Ticks`).
|
||||
- `$2` = the interval in ticks. Both are `NpgsqlDbType.Bigint`.
|
||||
- The result is exact because both sides are µs-aligned (R6), and it cannot overflow before year 9999.
|
||||
- **(as built)** The bucket timestamp returned for a Processed bucket is the *rounded* window start plus `i × interval`, so with a
|
||||
sub-µs request start it differs from SQL Server's label by the sub-µs remainder; values and membership are identical.
|
||||
- The query returns the full `BucketRow` statistics. Usable, good, bad and uncertain counts come from
|
||||
`sum(CASE …)` over the status column.
|
||||
- It does **not** use `SAMPLE BY`, whose alignment rules differ from ours. Computing the index ourselves keeps the
|
||||
bucket edges identical to SQL Server: start inclusive, end exclusive, a partial final bucket, interval 0, and the
|
||||
100k cap.
|
||||
- The time of the min or max value (ActualTime family) comes from a `row_number()` window function in the same single
|
||||
statement: `ORDER BY num_value ASC|DESC, ts ASC`, with ties going to the earliest sample (T0-confirmed; the form is
|
||||
below). No second statement is used. `AggregateEvaluator` then produces the values.
|
||||
- **Confirmed forms (T0 Q8).** They mirror `SqlAggregateQueries`'s three families and column order. They were checked
|
||||
against hand-computed seeds and a 60k-row randomized cross-check across 4 partitions: 617 buckets, 0 mismatches.
|
||||
- Classified subquery `s`:
|
||||
```sql
|
||||
SELECT (cast(ts AS LONG) * 10 - $1) / $2 AS b, ts, kind, num_value, int_value, str_value,
|
||||
CASE WHEN status < 2147483648 AND kind <> 0 THEN 1 ELSE 0 END AS u,
|
||||
CASE WHEN status < 1073741824 AND kind <> 0 THEN 1 ELSE 0 END AS g
|
||||
FROM {p}sample WHERE tag = $3 AND ts >= $4 AND ts < $5 AND NOT deleted
|
||||
```
|
||||
- Counts (ordinals 1–5 after `b`): `count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() -
|
||||
sum(u) AS bad_count, sum(CASE WHEN u = 1 AND kind IN (4, 5) THEN 1 ELSE 0 END) AS non_numeric_count`.
|
||||
- **Simple:** `SELECT b, <counts>, avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, min(CASE WHEN u = 1 THEN
|
||||
num_value END) AS min_value, max(…) AS max_value, sum(…) AS sum_value FROM (<s>) GROUP BY b ORDER BY b`.
|
||||
- **StartEnd:**
|
||||
```sql
|
||||
WITH s AS (<s>), st AS (SELECT b, <counts> FROM s GROUP BY b),
|
||||
f AS (SELECT b, kind, num_value, int_value, str_value, ts,
|
||||
row_number() OVER (PARTITION BY b ORDER BY ts) AS rf,
|
||||
row_number() OVER (PARTITION BY b ORDER BY ts DESC) AS rl FROM s WHERE u = 1)
|
||||
SELECT st.*, x.kind, x.num_value, x.int_value, x.str_value, x.ts, y.kind, y.num_value, y.int_value, y.str_value, y.ts
|
||||
FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b
|
||||
LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON y.b = st.b ORDER BY st.b
|
||||
```
|
||||
- **ActualTime:** the same `s`/`st`, then `r AS (SELECT b, num_value, ts, row_number() OVER (PARTITION BY b ORDER
|
||||
BY num_value ASC|DESC, ts ASC) AS rn FROM s WHERE u = 1 AND num_value IS NOT NULL) SELECT st.*, r.num_value, r.ts
|
||||
FROM st LEFT JOIN (SELECT * FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b`.
|
||||
- A bucket with no usable sample returns NULL times but **`kind = 0`** (BYTE has no NULL). The mapper keys absence on
|
||||
the NULL `ts`.
|
||||
- `first()/last()` and `arg_min/arg_max` also returned correct values, but no `ORDER BY` fixes their tie order. The
|
||||
window forms state the earliest-timestamp tie-break explicitly and are the ones to use.
|
||||
- **Errors.** A failed read fails only that node, under the same data-source contract as SQL Server.
|
||||
|
||||
## 4. Configuration (`ServerHistorian:QuestDb`)
|
||||
|
||||
| Key | Default | Validation |
|
||||
|---|---|---|
|
||||
| `ConnectionString` | — | Required when `Backend=QuestDb` and a consumer is enabled. An Npgsql string. **T0-confirmed:** `Host=questdb;Port=8812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading` (the docker-dev overlay from the host: `Host=localhost;Port=18812;…`). `ServerCompatibilityMode=NoTypeLoading` is kept: it skips the `pg_type` load, and every confirmed form was run with it. The string also works without it. `No Reset On Close` is **not** needed, because pooled open → close → reopen works. |
|
||||
| `TablePrefix` | `hist_` | `^[A-Za-z0-9_]+$`. Only checked when a consumer is enabled. |
|
||||
| `RetentionDays` | unset | ≥ 1 when set |
|
||||
| `CommandTimeout` | `00:00:30` | > 0 |
|
||||
| `AutoCreateSchema` | `true` | — |
|
||||
| `VisibilityTimeout` | `00:00:05` | ≥ 0. 0 disables the wait. |
|
||||
|
||||
The Gateway keys (`Endpoint`, `ApiKey`, `UseTls`) are not required for QuestDb, the same as for SqlServer.
|
||||
|
||||
**(as built) Pool size.** QuestDB's default limit is **64** PG-wire connections; Npgsql's default `Maximum Pool Size` is **100**, and the
|
||||
read, write, update and alarm paths share one pool per node. Operators should set `Maximum Pool Size` in the connection string below the
|
||||
server limit, per node (a redundant pair shares one QuestDB). The validator checks the prefix only when a consumer is enabled, but the
|
||||
connection helper also refuses a malformed prefix (`ArgumentException`) when the first seam is built.
|
||||
|
||||
## 5. Testing
|
||||
|
||||
- **T1 refactor gate.** The SQL Server integration suite (118) and unit suite pass unchanged before and after the
|
||||
extraction. Each moved piece of logic is falsified once against its existing test.
|
||||
- **Conformance suite** (`tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance`, a class library of abstract
|
||||
xUnit tests). Each backend's integration project subclasses it and supplies a fixture and an `IConformanceSeeder`,
|
||||
which writes rows directly in that backend's schema.
|
||||
- It covers raw forward, reverse and paging; AtTime, including the outage, after-last and future rules; all 13
|
||||
aggregates with the existing hand-computed literals; the quality contract; insert-if-absent writes and
|
||||
redelivery; the HistoryUpdate result codes; and over-long tag names.
|
||||
- Each legitimate backend difference is a named, documented override, never a silent skip. Microsecond rounding is
|
||||
the only expected one.
|
||||
- **QuestDB-only tests:**
|
||||
- a tombstone hides the row and a redelivery does not resurrect it;
|
||||
- the read-your-writes wait;
|
||||
- TTL is applied, verified through `tables()` metadata;
|
||||
- concurrent bootstrap from two instances;
|
||||
- intra-batch microsecond collisions;
|
||||
- the `hist_tag` upsert;
|
||||
- transient-failure behaviour, writing against a stopped port;
|
||||
- **(as built)** pre-1970 handling (writer drop, `BadOutOfRange`, read bounds), the `long.MinValue` read repair, Insert over a tombstone,
|
||||
per-tag updater serialization, and a forward cap-1 paging regression against a real QuestDB.
|
||||
- Conformance differences are named overrides: pre-1970 seeds are skipped on QuestDB, and AtTime timestamp assertions compare `Round(x)` to
|
||||
the requested time (identity on SQL Server). The QuestDB fixture prunes idle pool connections after 1 s because of the 64-connection limit.
|
||||
- **Fixture.** `HISTORIAN_QUESTDB_TEST_CONNECTION`; when it is absent, the tests skip. Each test class uses a unique
|
||||
throwaway `TablePrefix` (`t_<guid>_`) and drops its tables on dispose.
|
||||
- **Host.** Registry and validator tests for `Backend=QuestDb`, beside the SqlServer ones.
|
||||
|
||||
## 6. Environment
|
||||
|
||||
- QuestDB is an **optional overlay file**, `docker-dev/docker-compose.questdb.yml`, so the default rig is unchanged.
|
||||
It defines the `questdb` service:
|
||||
- image `questdb/questdb:10.0.1`, pinned by T0 and pushed 2026-08-24;
|
||||
- container `otopcua-dev-questdb-1`;
|
||||
- started with `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d
|
||||
--no-deps questdb`.
|
||||
- It exposes PG wire on host port **18812** (→ 8812) and the web console/REST on host port **19000** (→ 9000), so it
|
||||
does not collide with AdminUI `:9000`/`:9200`.
|
||||
- It uses the named volume `otopcua-questdb-data`.
|
||||
- The central nodes gain `ServerHistorian__QuestDb__ConnectionString` (`OTOPCUA_HISTORIAN_QUESTDB`), defaulting to the
|
||||
`questdb` service.
|
||||
- No new NuGet packages.
|
||||
|
||||
## 7. Live gate (T9, run by the controller)
|
||||
|
||||
On docker-dev, with `Backend=QuestDb`:
|
||||
1. Only the Primary records, with real kinds and status, one row per value.
|
||||
2. Raw and Processed HistoryRead return identical results through the raw and UNS NodeIds, and match a direct
|
||||
QuestDB query.
|
||||
3. HistoryUpdate covers insert, duplicate, replace, replace-missing and delete-at-time, each confirmed by a read and
|
||||
by a direct query. After a delete, an outbox redelivery does not bring the row back. A WriteOperate-only key and an
|
||||
anonymous session are both denied.
|
||||
4. On failover, recording continues at 1×.
|
||||
5. Modbus device loss stores a `BadCommunicationError` Empty sample, then the `BadNoCommunication` outage marker,
|
||||
then Good on recovery.
|
||||
6. TTL is visible in table metadata when `RetentionDays` is set.
|
||||
|
||||
Evidence goes in `docs/plans/2026-10-10-questdb-historian-live-gate.md`, and the rig is restored afterwards.
|
||||
|
||||
## 8. Work order
|
||||
|
||||
| Wave | Task | Model |
|
||||
|---|---|---|
|
||||
| W0 | **T0 spike.** Pin the image, add the compose service, and prove every QuestDB assumption here: Npgsql parameterised multi-row inserts, `DEDUP UPSERT`, `wal_tables()` visibility, TTL syntax, microsecond timestamps over Npgsql, the min/max-time query form, AtTime neighbour lookup, concurrent `CREATE TABLE IF NOT EXISTS`. **Amend §2–§4 with the findings before W1.** | controller + sonnet |
|
||||
| W1 | T1: extract `Sql.Common`, leaving SQL Server's behaviour unchanged | opus |
|
||||
| W2 | T2 conformance suite over SQL Server ‖ T3 QuestDB skeleton (options, connection, bootstrapper, fixture) | sonnet ‖ sonnet |
|
||||
| W3 | T4 writes, provisioning and alarms ‖ T5 reads and aggregates ‖ T6 updater with soft delete | sonnet ‖ opus ‖ sonnet |
|
||||
| W4 | T7 provider, registry and validator, plus the conformance suite run against QuestDB | sonnet |
|
||||
| W5 | T8 docs (Historian.md, Configuration.md, CLAUDE.md, docker-dev README), then a Fable whole-branch review, one fix wave, and T9 live gate | sonnet / fable / opus / controller |
|
||||
@@ -0,0 +1,205 @@
|
||||
# QuestDB historian — live gate (T9), 2026-10-11
|
||||
|
||||
**Verdict: PASSED.** All six legs of spec §7 passed on the docker-dev rig. The rig ran `feat/questdb-historian` at
|
||||
`2ebe0a42`, with the central pair on image `0701f6a833ea`, rebuilt for this gate. The QuestDB backend pointed at the
|
||||
overlay `questdb` service (`questdb/questdb:10.0.1`, `docker-dev/docker-compose.questdb.yml`), with default table
|
||||
prefix `hist_`.
|
||||
|
||||
The gate also produced two findings, neither in the QuestDB backend. Both are described under "Gate findings" below:
|
||||
- A crash failover (SIGKILL of the recording Primary) leaves a recording gap of about 30 s. This is the #537
|
||||
Primary-gated capture waiting for Akka downing; it is backend-independent.
|
||||
- One `libgssapi_krb5.so.2` load error is printed per central boot. It is benign.
|
||||
|
||||
## Setup
|
||||
|
||||
- `docker-dev/docker-compose.yml` gained `ServerHistorian__QuestDb__RetentionDays`
|
||||
(`OTOPCUA_HISTORIAN_QUESTDB_RETENTION_DAYS`, default empty = no TTL) on both centrals, beside the existing
|
||||
`ServerHistorian__QuestDb__ConnectionString` (`OTOPCUA_HISTORIAN_QUESTDB`).
|
||||
- Central-1 and central-2 were started with:
|
||||
|
||||
```
|
||||
OTOPCUA_HISTORIAN_ENABLED=true OTOPCUA_HISTORIAN_BACKEND=QuestDb OTOPCUA_CONTHIST_ENABLED=true \
|
||||
docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d --no-deps central-1 central-2
|
||||
```
|
||||
|
||||
With `RetentionDays` empty, both nodes booted cleanly; the empty value binds as "no TTL". Each logged
|
||||
`QuestDB historian connection pool: Maximum Pool Size = 24 (default cap).` (the final-review I-1 fix).
|
||||
- Cluster `MAIN` fixtures on `10.100.0.35`: opc-plc (`opcplc/plc/FastUInt1`, UInt32, about 1 Hz) and pymodbus
|
||||
`standard` (`pymodbus/plc/HR5`).
|
||||
- Gate configuration, written to the config DB and deployed headlessly (`deploy.sh`, deployment `0b4f5208`):
|
||||
- New Calculation String tag `Calc/eng/FastLabel` = `"v" + FastUInt1` (script `MAIN-script-fastlabel`).
|
||||
- `FastUInt1`, `FastLabel` and `HR5` were marked `isHistorized`.
|
||||
- A UNS reference `a/l/e1/FastUInt1` was added.
|
||||
- Tags were provisioned at deploy (`dispatched=3, requested=3, ensured=3, skipped=0, failed=0` on both nodes). `hist_tag`:
|
||||
|
||||
| name | data_type |
|
||||
|---|---|
|
||||
| `Calc/eng/FastLabel` | 9 (String) |
|
||||
| `opcplc/plc/FastUInt1` | 5 (Int64) |
|
||||
| `pymodbus/plc/HR5` | 4 |
|
||||
|
||||
- Tables at first boot, from `tables()`:
|
||||
- `hist_sample`, `hist_event`: WAL, `PARTITION BY DAY`, `ttlValue 0`.
|
||||
- `hist_tag`: WAL, `PARTITION BY YEAR`.
|
||||
- Two data-plane API keys were minted for the gate, each with a `Cluster`-scoped grant. The ACL flag stayed off, which
|
||||
is the rig default.
|
||||
- `gate-historyupdate`: ReadOnly | HistoryUpdate (4239)
|
||||
- `gate-writeop`: ReadOnly | WriteOperate (159)
|
||||
|
||||
Direct queries used QuestDB REST: `curl -G localhost:19000/exec --data-urlencode "query=…"`.
|
||||
|
||||
## 1. Recording: Primary-only, typed, real status, one row per value
|
||||
|
||||
- **Rows carry real kinds and statuses:**
|
||||
- `kind=2` Int64 `int_value 278` (with `num_value 278.0`), status `0`.
|
||||
- `kind=4` String `v278`, status `0`.
|
||||
- At startup, `kind=0` Empty with `2150760448` (`0x80320000` BadWaitingForInitialData) and `2150694912`
|
||||
(`0x80310000` BadNoCommunication). These match the SQL Server gate.
|
||||
- **Primary-only:**
|
||||
- Central-2 was the recording Primary, and its outbox `hlog.log.0` grew.
|
||||
- Central-1 logged "this node is not the recording Primary … suppressing value capture". Its `hlog.log.0` kept its
|
||||
2026-10-09 19:13 mtime and was not written.
|
||||
- **One row per value**, from 07:43:20 onwards:
|
||||
- 57 FastUInt1 rows over 57 distinct timestamps; 57 FastLabel rows over 57.
|
||||
- All 57 pair one-to-one: same `ts`, and `str_value = 'v' + int_value`.
|
||||
- **Outbox carry-over across a backend switch.** Two rows carry source time `2026-10-09T19:13:40.129127Z` (`296` /
|
||||
`v296`) with `server_ts 07:42:27`, before the deploy. They were un-drained entries left in central-2's outbox by the
|
||||
2026-10-09 SQL Server gate, and they drained into QuestDB at boot. An outbox entry outlives the backend it was
|
||||
captured under, by design: the outbox is backend-neutral.
|
||||
|
||||
## 2. HistoryRead through the raw and UNS NodeIds
|
||||
|
||||
All reads went to central-1 (`:4840`).
|
||||
|
||||
- **Raw read**, 07:44:00–07:44:10, on `ns=2;s=opcplc/plc/FastUInt1` and on `ns=3;s=a/l/e1/FastUInt1`:
|
||||
- Both return the identical 10 rows, `271`…`280`, all Good.
|
||||
- They equal the direct query `select ts, int_value, status from hist_sample where tag=… and not deleted` exactly,
|
||||
at microsecond precision (for example `07:44:00.952990Z`).
|
||||
- **Average at 10 s**, 07:44:00–07:44:30, through both NodeIds:
|
||||
- Both return `275.5 / 285.5 / 295.5`, all Good.
|
||||
- The direct `avg(int_value)` per 10 s bucket gives `275.5 / 285.5 / 295.5`, over 10, 10 and 10 samples.
|
||||
|
||||
## 3. HistoryUpdate
|
||||
|
||||
These calls went to central-1 on `ns=2;s=opcplc/plc/FastUInt1`, with `-U apikey`. Each was checked by `historyread`
|
||||
and by a direct query.
|
||||
|
||||
| Call | Result | Confirmed |
|
||||
|---|---|---|
|
||||
| insert 06:00:00 = 42 (`gate-historyupdate`) | `GoodEntryInserted` | row `42`, status 0, `deleted=false` |
|
||||
| repeat insert = 43 | `BadEntryExists` | row still `42` |
|
||||
| replace 06:00:00 = 99 | `GoodEntryReplaced` | row `99` |
|
||||
| replace 06:00:30 (no entry) | `BadNoEntryExists` | — |
|
||||
| update 06:01:00 = 11 (absent) | `GoodEntryInserted` | row `11` |
|
||||
| delete-at-time 06:00:00 | `Good` | tombstone row: `int_value NULL`, status `0x809B0000`, `deleted=true` |
|
||||
| insert over the tombstone, 06:00:00 = 55 | `GoodEntryInserted` (ledger ruling: an insert over a tombstone is an insert) | read returns `55` |
|
||||
| delete-at-time 06:00:00 again | `Good` | read returns only `06:01:00 = 11` |
|
||||
| insert with `gate-writeop` (WriteOperate only) | Node result `BadUserAccessDenied` | no row |
|
||||
| insert, anonymous | Node result `BadUserAccessDenied` | no row |
|
||||
| insert at 1969-12-31 (RC1) | `BadOutOfRange` | no row |
|
||||
|
||||
No QuestDB `UPDATE` or `DELETE` was issued: every change above is a new row under `DEDUP UPSERT KEYS(ts, tag)`.
|
||||
|
||||
### A delete is not resurrected by an outbox redelivery
|
||||
|
||||
This was proven with a real replay, not a simulation:
|
||||
1. **07:47:26–07:47:35:** `docker stop` QuestDB. The Primary (central-1) kept capturing; its writes failed, so the
|
||||
entries stayed pending in its outbox.
|
||||
2. **07:47:35:** the outbox was snapshotted while QuestDB was down (commit point `commit.1303.0`).
|
||||
3. **07:47:35:** QuestDB was restarted. The 16-value backlog (`476`…`491`) drained as one batch (`server_ts
|
||||
07:47:41.620267`), with no gap. This is a second result: QuestDB loss is ridden out by the outbox.
|
||||
4. **07:50:45–46:** `delete-at-time` on two backlog rows, `07:47:28.950591` (`479`) and `07:47:31.949216` (`482`).
|
||||
Both became tombstones (`deleted=true`, `0x809B0000`).
|
||||
5. **07:50:46:** central-1 was stopped. Its outbox directory (on the `otopcua-localdb-central-1` volume) was replaced
|
||||
with the pre-delivery snapshot.
|
||||
6. **07:51:15:** central-1 was started. It re-drained the snapshot's entries, and the outbox commit point advanced
|
||||
`1303 → 1328`.
|
||||
|
||||
After the replay:
|
||||
- Both tombstones are unchanged: same `server_ts 07:50:45.669146 / 07:50:46.074579`, still `deleted=true`.
|
||||
- Every other row in the window kept its original `server_ts 07:47:41.620267`, so nothing was re-written.
|
||||
- 24 rows over 12 keys (2 tags × 12): no duplicates.
|
||||
- `historyread` 07:47:27–07:47:33 returns `478, 480, 481, 483`; `479` and `482` stay deleted.
|
||||
|
||||
A first attempt used `ALTER TABLE hist_sample SUSPEND WAL` plus a SIGKILL of the Primary. It could not produce a
|
||||
redelivery. The writer deliberately treats a suspended table as committed and acknowledges it (R5: rows committed to
|
||||
the WAL become visible later, and are not re-sent). It logged "QuestDB table hist_sample is suspended (writerTxn 327
|
||||
< sequencerTxn 337); not waiting" and acked. That attempt exposed the crash-failover finding below.
|
||||
|
||||
## 4. Failover
|
||||
|
||||
- **07:50:46:** central-1 was stopped (step 5 above). Central-2 logged "now the recording Primary" at 07:50:46.
|
||||
- **07:51:57:** `docker stop` central-2, then the Primary. Central-1 logged "this node is now the recording Primary …
|
||||
resuming value capture" at 07:51:57, as central-2 began leaving.
|
||||
- **07:52:13:** central-2 was restarted and rejoined as standby ("not the recording Primary"). Its outbox
|
||||
`hlog.log.0` stayed frozen at 57856 bytes (mtime 07:51:58). Central-1's grew from 65536 to 84480.
|
||||
- **Rate across both failovers was 1×.** Per-30 s FastUInt1 counts from 07:50:00 to 07:54:30:
|
||||
30, 29, 30, 30, 30, 30, 29, 29, 30.
|
||||
- There was no gap at the boundary: `743`…`754` are contiguous across 07:51:57.
|
||||
- There was no doubling: 267 rows over 267 distinct timestamps.
|
||||
|
||||
## 5. Device loss: driver Bad sample and comms-loss outage marker
|
||||
|
||||
This leg ran on Modbus (`HR5`). `otopcua-pymodbus-standard` was stopped at 07:54:40 and restarted at 07:56:51.
|
||||
|
||||
| ts | kind | value | status | server_ts |
|
||||
|---|---|---|---|---|
|
||||
| 07:51:18.715101 | 2 | 5 | `0` | 07:51:18 |
|
||||
| 07:54:51.960244 | 0 (Empty) | NULL | `0x80050000` BadCommunicationError (the driver's own Bad publish) | 07:56:13 |
|
||||
| 07:56:48.652185 | 0 (Empty) | NULL | `0x80310000` BadNoCommunication (the #527 watchdog outage marker) | 07:56:48 |
|
||||
| 07:56:58.688150 | 2 | 5 | `0` (recovery re-publish) | 07:56:58 |
|
||||
|
||||
Reads over the outage:
|
||||
- Raw `historyread` 07:50–07:58 shows Good, Bad, Bad, Good.
|
||||
- Average at 120 s, 07:52–07:58, gives `(null) Bad` / `(null) Bad` / `5 Uncertain`. The outage is visible and is not
|
||||
bridged.
|
||||
|
||||
The Bad sample's `server_ts` is 82 s after its source time. Its source time is the driver's failed-read stamp; it
|
||||
reached the recorder later. Its position in the series is correct either way, because ordering and reads key on
|
||||
source time.
|
||||
|
||||
## 6. TTL visible in table metadata
|
||||
|
||||
Both centrals were recreated with `OTOPCUA_HISTORIAN_QUESTDB_RETENTION_DAYS=30`. At 07:57:41 the bootstrapper logged
|
||||
`Applied TTL 30 day(s) to QuestDB table hist_sample.` and the same for `hist_event`. The TTL was applied to the
|
||||
already existing tables.
|
||||
|
||||
`tables()` before and after:
|
||||
|
||||
| table | before | after |
|
||||
|---|---|---|
|
||||
| `hist_event` | 0 HOUR | 30 DAY |
|
||||
| `hist_sample` | 0 HOUR | 30 DAY |
|
||||
| `hist_tag` | 0 HOUR | 0 HOUR (no TTL on the tag catalog, by design) |
|
||||
|
||||
`SHOW CREATE TABLE hist_sample`:
|
||||
`… timestamp(ts) PARTITION BY DAY TTL 30 DAYS DEDUP UPSERT KEYS(ts,tag);`
|
||||
|
||||
Recording resumed after the restart: 57 FastUInt1 rows and 56 FastLabel rows in the first minute.
|
||||
|
||||
## Gate findings
|
||||
|
||||
1. **A crash failover leaves a recording gap of about 30 s. This is backend-independent and predates this work.**
|
||||
- **What happened:** after `docker kill` (SIGKILL) of the recording Primary at 07:45:58, the next FastUInt1 row
|
||||
is at 07:46:29. Central-1 became the recording Primary only at 07:46:30, when the killed node rejoined as a new
|
||||
incarnation.
|
||||
- **Why:** #537 gates capture to the Primary. A crashed Primary stays unreachable but not yet downed for the Akka
|
||||
failure-detector window plus `auto-down-unreachable-after = 15s`, so no node captures during that time.
|
||||
- **Contrast:** a clean `docker stop` hands over at once (leg 4). The SQL Server gate only exercised the clean
|
||||
path.
|
||||
- **Status:** proposed follow-up issue, not fixed here.
|
||||
2. **`Cannot load library libgssapi_krb5.so.2` is printed once per central boot.** It does not appear on the site
|
||||
nodes, which have neither a ConfigDb nor a QuestDB client. It is most likely Npgsql's default GSS-encryption probe,
|
||||
which falls back cleanly. Everything worked. If it is confirmed as Npgsql, `GSS Encryption Mode=Disable` in the
|
||||
connection string silences it.
|
||||
|
||||
## Restore
|
||||
|
||||
- The rig was restored afterwards:
|
||||
- central-1 and central-2 were recreated with the compose defaults (historian off, `Backend=Gateway`).
|
||||
- The gate tag, script, UNS reference, API keys and ACL rows were removed.
|
||||
- The `HR5` and `FastUInt1` TagConfigs were reverted.
|
||||
- The configuration was redeployed.
|
||||
- The overlay `questdb` container was stopped and removed; the `otopcua-questdb-data` volume was kept.
|
||||
- The opc-plc and pymodbus fixtures were stopped (normally down).
|
||||
- The compose keys stay: `Backend` defaults to `Gateway`, and `RetentionDays` defaults to empty.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
@@ -0,0 +1,451 @@
|
||||
# QuestDB historian backend — T0 feasibility spike
|
||||
|
||||
**Status:** run 2026-10-11 against a live container. Gitea #666. Spec: `docs/plans/2026-10-10-questdb-historian-design.md`
|
||||
(§2–§4 amended in place from this doc; every confirmed form there is marked **T0-confirmed**).
|
||||
|
||||
## Fixture
|
||||
|
||||
| Item | Value |
|
||||
|---|---|
|
||||
| Image | **`questdb/questdb:10.0.1`** — Docker Hub `tag_last_pushed` **2026-08-24T14:00:08Z** (48 days before 2026-10-10) |
|
||||
| Why that tag | Newest exact `x.y.z` tag ≥ 14 days older than 2026-10-10. Newer tags are `nightly*` / `patch-*` only. Floor for TTL is **8.2.3**; 10.0.1 is well above it. |
|
||||
| Build | `SELECT build()` → `Build Information: QuestDB 10.0.1, JDK 25.0.2, Commit Hash 7a391566ac827e0d8c269b91f2415c16b0a32a84` |
|
||||
| Overlay | `docker-dev/docker-compose.questdb.yml` — container `otopcua-dev-questdb-1`, ports `18812:8812` (PG wire) and `19000:9000` (console/REST), named volume `otopcua-questdb-data` (explicit `name:` so it is not project-prefixed) |
|
||||
| Up | `docker compose -f docker-dev/docker-compose.yml -f docker-dev/docker-compose.questdb.yml up -d --no-deps questdb` (a `--dry-run` showed only the volume + `otopcua-dev-questdb-1` being created; nothing else in the `otopcua-dev` project is touched) |
|
||||
| Healthcheck | the image ships `/usr/bin/curl` (`command -v curl` → `/usr/bin/curl`; no `wget`), so `curl -fs http://localhost:9003/status` is kept. Container reached `healthy`. |
|
||||
| Harness | throwaway net10.0 console app, Npgsql **10.0.3** (the repo's central pin, `Directory.Packages.props` line 97), in the session scratchpad `questdb-spike/` with its own `Directory.Build.props`, `Directory.Packages.props` and `nuget.config`. Nothing was added to the repo. |
|
||||
|
||||
### Connection string (confirmed)
|
||||
|
||||
```
|
||||
Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading
|
||||
```
|
||||
|
||||
Inside the docker network the same string uses `Host=questdb;Port=8812`.
|
||||
|
||||
## Q1 — keywords, parameter types, CLR types
|
||||
|
||||
```
|
||||
[OK] open+SELECT 1 with 'Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb'
|
||||
[OK] open+SELECT 1 with '...;ServerCompatibilityMode=NoTypeLoading'
|
||||
[OK] open->close->reopen x5 on one data source (no 'No Reset On Close')
|
||||
[OK] reopen after a parameterised INSERT (prepared-ish state)
|
||||
[FAIL] ts Utc + Timestamp: ArgumentException: Cannot write DateTime with Kind=UTC to PostgreSQL type 'timestamp without time zone', consider using 'timestamp with time zone'. ...
|
||||
[OK] ts Utc + TimestampTz
|
||||
[OK] ts Utc + inferred
|
||||
[OK] ts Unspecified + Timestamp
|
||||
[FAIL] ts Local + TimestampTz: ArgumentException: Cannot write DateTime with Kind=Local to PostgreSQL type 'timestamp with time zone', only UTC is supported. ...
|
||||
[OK] kind bound as Integer (int4)
|
||||
[OK] tag bound as Text
|
||||
[OK] NaN double insert
|
||||
[OK] +Inf double insert
|
||||
cols: ts:timestamp without time zone/DateTime, tag:character varying/String, kind:smallint/Int16, num_value:double precision/Double,
|
||||
int_value:bigint/Int64, str_value:character varying/String, status:bigint/Int64, deleted:boolean/Boolean, server_ts:timestamp without time zone/DateTime
|
||||
2026-01-02T03:04:05.1234560(Unspecified) | ts Utc + TimestampTz | 3<Int16> | 1.5 | 7<Int64> | s | 2147483648<Int64> | False | 2026-10-11T04:56:27.1103710(Unspecified)
|
||||
...
|
||||
2026-01-02T03:04:11.1234560(Unspecified) | nan | 0<Int16> | NULL | NULL | NULL | NULL | False | NULL
|
||||
2026-01-02T03:04:12.1234560(Unspecified) | inf | 0<Int16> | NULL | NULL | NULL | NULL | False | NULL
|
||||
integer arithmetic with $n:
|
||||
cols: ts::LONG - $1:bigint/Int64 [OK] cast(ts AS LONG) - $1 (Bigint)
|
||||
cols: (ts::LONG * 10 - $1) / $2:bigint/Int64 [OK] (Bigint) [OK] (Integer)
|
||||
```
|
||||
|
||||
Sentinel NULLs (extra check):
|
||||
|
||||
```
|
||||
cols: ts, l:bigint/Int64, i:integer/Int32, b:smallint/Int16, z:boolean/Boolean, l = NULL:boolean
|
||||
2026-03-01T00:00:00 | NULL | NULL | 0 | False | True <- l = long.MinValue, i = int.MinValue written
|
||||
2026-03-01T00:00:01 | -9223372036854775807 | -2147483647 | 0 | False | False
|
||||
2026-03-01T00:00:02 | NULL | NULL | 0 | False | True <- BYTE/BOOLEAN bound as DBNull
|
||||
```
|
||||
|
||||
**Decisions**
|
||||
- Both connection strings work. **Keep `ServerCompatibilityMode=NoTypeLoading`**: it skips the `pg_type` load, and every
|
||||
form in this doc was run with it on. **`No Reset On Close` is not needed**: open → close → reopen on one pooled
|
||||
`NpgsqlDataSource` works.
|
||||
- **Timestamps are bound as `NpgsqlDbType.TimestampTz` with `DateTimeKind.Utc`.** `Timestamp` + `Utc` throws in Npgsql
|
||||
before reaching the server. `Unspecified` + `Timestamp` also works, but one rule is enough.
|
||||
- **Timestamps read back as `DateTime` with `Kind=Unspecified`**, wall clock = UTC. The reader must
|
||||
`DateTime.SpecifyKind(r.GetDateTime(i), DateTimeKind.Utc)`.
|
||||
- CLR types returned: `BYTE` → `smallint` / **`Int16`** (`GetInt16`); `SYMBOL` → `character varying` / `String`;
|
||||
`VARCHAR` → `String`; `LONG` → `Int64`; `INT` → `Int32`; `BOOLEAN` → `Boolean`; `TIMESTAMP` → `DateTime`;
|
||||
`DOUBLE` → `Double`. `count()`/`sum(int)` → `Int64`.
|
||||
- Parameter types: `BYTE` ← `Smallint` (`Integer` also accepted); `SYMBOL`/`VARCHAR` ← `Varchar` (`Text` also accepted);
|
||||
`LONG` ← `Bigint`; `BOOLEAN` ← `Boolean`; `DOUBLE` ← `Double`; a NULL is `DBNull.Value` with the column's
|
||||
`NpgsqlDbType`.
|
||||
- `$n` in integer arithmetic: **bind as `NpgsqlDbType.Bigint`** (Integer also worked for small values; Bigint is the
|
||||
rule because tick offsets exceed int32).
|
||||
- **DOUBLE NaN and ±Inf read back as NULL.** **`LONG` `long.MinValue` and `INT` `int.MinValue` are QuestDB's NULL
|
||||
sentinels** and read back NULL. **`BYTE` and `BOOLEAN` have no NULL**: unset or DBNull reads `0` / `false`. So
|
||||
`deleted` defaults to `false` and `kind` of a never-written field is `0` (Empty).
|
||||
- **Every server error carries `SqlState = 00000`.** Classification cannot use SQLSTATE: a `PostgresException` is a
|
||||
server-side rejection (decide by `MessageText`), any other `NpgsqlException`/`IOException`/`SocketException`/timeout
|
||||
is transport.
|
||||
|
||||
## Q2 — multi-row parameterised INSERT and batch size
|
||||
|
||||
```
|
||||
[OK] 1000 rows x 9 params = 9000 params 88 ms
|
||||
[OK] 2000 rows x 9 params = 18000 params 167 ms
|
||||
[OK] 3640 rows x 9 params = 32760 params 342 ms
|
||||
[FAIL] 3641 rows x 9 params = 32769 params: PostgresException: 00000: invalid parameter count [parameterCount=-32767, offset=225670
|
||||
[FAIL] 5000 rows x 9 params = 45000 params: PostgresException: 00000: invalid parameter count [parameterCount=-20536, ...
|
||||
[FAIL] 7282 rows x 9 params = 65538 params: NpgsqlException: A statement cannot have more than 65535 parameters
|
||||
20 x 1000-row statements: fails=0, 122 ms total
|
||||
count()=27251 | count_distinct(ts)=27251 expected rows = 27251
|
||||
```
|
||||
|
||||
**Decision:** QuestDB reads the Bind parameter count as a **signed int16**, so the real limit is **32,767 parameters per
|
||||
statement**, not Npgsql's 65,535. **Batch size = 1,000 rows per `INSERT … VALUES (…),(…)`** (9,000 parameters for
|
||||
`hist_sample`), which is reliable and far under the limit. **DeleteRaw page size = 1,000 rows** (see the DDL section).
|
||||
|
||||
## Q3 — `DEDUP UPSERT KEYS`
|
||||
|
||||
```
|
||||
before: 2026-03-01T00:00:00 | a | 1 | False 2026-03-01T00:00:00 | b | 2 | False
|
||||
after re-insert of (t0,'a') as tombstone:
|
||||
2026-03-01T00:00:00 | a | NULL | True 2026-03-01T00:00:00 | b | 2 | False
|
||||
NOT deleted: 2026-03-01T00:00:00 | b | 2
|
||||
same key twice in one statement (10 then 20): 2026-03-01T00:00:01 | c | 20
|
||||
hist_event with event_id VARCHAR as a dedup key:
|
||||
[OK] CREATE hist_event-shape with DEDUP UPSERT KEYS(ts, event_id) on VARCHAR
|
||||
[OK] insert e1 'first', e2, then e1 'second'
|
||||
2026-03-01T00:00:00 | e1 | src1 | 500 | second | NULL
|
||||
2026-03-01T00:00:00 | e2 | src1 | 500 | other | NULL
|
||||
```
|
||||
|
||||
**Decision:** re-inserting a key replaces the whole row; a tombstone written at `(ts, tag)` hides the row from `NOT
|
||||
deleted`. **Within one statement the LAST duplicate wins**, so the writer's "first value wins" rule must be applied
|
||||
client-side before the INSERT (spec §2.2 step 1 already does this). **`event_id VARCHAR` works as a dedup key**, so the
|
||||
S4 fallback (no dedup on `hist_event`) is **not needed**.
|
||||
|
||||
## Q4 — `wal_tables()` and the visibility wait
|
||||
|
||||
```
|
||||
cols: name:varchar, suspended:boolean, writerTxn:bigint, bufferedTxnSize:bigint, sequencerTxn:bigint,
|
||||
errorTag:varchar, errorMessage:varchar, memoryPressure:integer
|
||||
run 0: seqTxn before=0 after=1 visibleImmediately=False waitMs=13 visibleAfterWait=True
|
||||
run 1: seqTxn before=1 after=2 visibleImmediately=False waitMs=0 visibleAfterWait=True
|
||||
...
|
||||
run 16: seqTxn before=16 after=17 visibleImmediately=False waitMs=6 visibleAfterWait=True
|
||||
lag ms: min=0 median=0 p95=6 max=13; visible-without-wait 17/20
|
||||
```
|
||||
|
||||
(An earlier run: visible without waiting 13/20, max 6 ms.) Every autocommitted INSERT is exactly one sequencer
|
||||
transaction.
|
||||
|
||||
**Decision (the form T3's `QuestDbWalVisibility` implements):**
|
||||
1. After the INSERT(s) return, on the same or any connection:
|
||||
`SELECT sequencerTxn FROM wal_tables() WHERE name = $1` (`$1` = the table name, `Varchar`) → `target`.
|
||||
2. Poll `SELECT writerTxn, suspended FROM wal_tables() WHERE name = $1` every ~10 ms until `writerTxn >= target`, or
|
||||
`suspended` is true, or `VisibilityTimeout` elapses.
|
||||
|
||||
A concurrent writer can only make `target` larger, which lengthens the wait slightly and never shortens it. The wait is
|
||||
needed: up to 7/20 inserts were not yet visible on the next query. The S11 key-poll fallback is **not needed**.
|
||||
|
||||
## Q5 — TTL
|
||||
|
||||
```
|
||||
[FAIL] CREATE ... PARTITION BY DAY WAL DEDUP UPSERT KEYS(ts, tag) TTL 7 DAYS (TTL last): unexpected token [TTL]
|
||||
[OK] CREATE ... PARTITION BY DAY TTL 7 DAYS WAL DEDUP UPSERT KEYS(ts, tag) (TTL before WAL)
|
||||
spike_q5 | DAY | True | True | ttlValue=1 | ttlUnit=WEEK
|
||||
[FAIL] ALTER TABLE spike_q5 SET TTL $1 DAYS (parameter): invalid syntax, should be <number> <unit> but was $1
|
||||
[FAIL] TTL 7 DAYS on PARTITION BY YEAR table: TTL value must be an integer multiple of the partition size (its time interval)
|
||||
Q5b ALTER ... SET TTL visibility in tables()
|
||||
SET TTL 30 DAYS : seqTxn 6->7; applied after 100 ms; tables() now ttlValue=30 ttlUnit=DAY
|
||||
SET TTL 14 DAYS : seqTxn 7->8; applied after 94 ms; tables() now ttlValue=2 ttlUnit=WEEK
|
||||
SET TTL 1 DAY : seqTxn 8->9; applied after 59 ms; tables() now ttlValue=1 ttlUnit=DAY
|
||||
SET TTL 0 DAYS : seqTxn 9->10; applied after 25 ms; tables() now ttlValue=0 ttlUnit=HOUR
|
||||
SET TTL 3 DAYS : seqTxn 10->11; applied after 60 ms; tables() now ttlValue=3 ttlUnit=DAY
|
||||
after inserting now-20d and now into a TTL 7 DAYS table:
|
||||
new | 2026-10-11T05:03:03
|
||||
Q5c TTL reference clock (future-dated row):
|
||||
after inserting a now+30d row, then one more ordinary commit:
|
||||
d-3 | 2026-10-08 now | 2026-10-11 now+1h | 2026-10-11 d+30 | 2026-11-10
|
||||
```
|
||||
|
||||
Image `conf/server.conf`: `#cairo.ttl.use.wall.clock=true` — "when true (default), TTL enforcement uses wall clock
|
||||
time to prevent accidental data loss when future timestamps are inserted".
|
||||
|
||||
**Decision:**
|
||||
- Create syntax: **`… TIMESTAMP(ts) PARTITION BY DAY TTL n DAYS WAL DEDUP UPSERT KEYS(…)`** — TTL goes **between
|
||||
`PARTITION BY` and `WAL`**. Alter: **`ALTER TABLE t SET TTL n DAYS`**; remove: **`ALTER TABLE t SET TTL 0 DAYS`**.
|
||||
`n` cannot be a parameter; it is the validated integer `RetentionDays`, formatted with `InvariantCulture`.
|
||||
- Metadata: `SELECT ttlValue, ttlUnit FROM tables() WHERE table_name = $1`. **QuestDB normalises the unit**: 7 days
|
||||
shows `1 WEEK`, 14 days `2 WEEK`, 30 days `30 DAY`, none `0 HOUR`. The bootstrapper compares in hours
|
||||
(`HOUR`=1, `DAY`=24, `WEEK`=168; any other unit counts as different) and issues `ALTER` only when they differ.
|
||||
- `ALTER … SET TTL` is a WAL transaction: `tables()` shows the old value until it is applied (25–100 ms). Do not
|
||||
re-read to verify without the Q4 wait.
|
||||
- Partitions older than TTL are dropped on commit. With the default wall-clock mode a future-dated sample does **not**
|
||||
expire current data.
|
||||
- TTL cannot go on the `PARTITION BY YEAR` `hist_tag` table; it is never needed there.
|
||||
|
||||
## Q6 — timestamp precision
|
||||
|
||||
```
|
||||
raw sent 2026-03-01T12:00:00.1234567(Utc) -> read 2026-03-01T12:00:00.1234560 micros=1772366400123456 exact=False
|
||||
raw9 sent 2026-03-01T12:00:01.1234569(Utc) -> read 2026-03-01T12:00:01.1234560 micros=1772366401123456 exact=False
|
||||
pre sent 2026-03-01T12:01:00.1234560(Utc) -> read 2026-03-01T12:01:00.1234560 micros=1772366460123456 exact=True
|
||||
```
|
||||
|
||||
**Decision:** Npgsql **truncates** sub-µs ticks (…1234567 → …123456; not rounded). That is truncation **toward zero
|
||||
relative to 2000-01-01**, so before 2000 it rounds *up* (Q13). A pre-truncated (`ticks - ticks % 10`) value
|
||||
round-trips exactly. This confirms R6: always `Round` first.
|
||||
|
||||
## Q7 — concurrent `CREATE TABLE IF NOT EXISTS`
|
||||
|
||||
```
|
||||
Q7 concurrent CREATE TABLE IF NOT EXISTS (4 connections x 20 rounds)
|
||||
errors: none
|
||||
plain CREATE on existing: SqlState=00000 MessageText='table already exists'
|
||||
```
|
||||
|
||||
(An earlier run with 2 connections × 20 rounds also had no errors.) **Decision:** `IF NOT EXISTS` was race-free here.
|
||||
The S11 catch rule stays as defence: catch a `PostgresException` whose `MessageText` contains **`already exists`**,
|
||||
then re-check with `SELECT count() FROM tables() WHERE table_name = $1`, and rethrow when the table is not there. Never
|
||||
key on SQLSTATE (it is always `00000`).
|
||||
|
||||
## Q8 — bucket statistics
|
||||
|
||||
Seed (3 × 10 s buckets from `start = t0 + 0.5 s`): bucket 0 holds Good 5, Uncertain 2, Bad 9, a usable String, Good 2
|
||||
(a min tie with the Uncertain 2), a tombstone of 100, and Good 7 at the last µs of the bucket. Bucket 1 holds Good 1 at
|
||||
the edge, an Empty `BadNoCommunication` and an Empty with Good status. Bucket 2 holds Bad 3 only. Another tag sits in
|
||||
range.
|
||||
|
||||
```
|
||||
Simple family (GROUP BY alias):
|
||||
cols: b, total_count, usable_count, good_count, bad_count, non_numeric_count, avg_value, min_value, max_value, sum_value
|
||||
0 | 6 | 5 | 4 | 1 | 1 | 4 | 2 | 7 | 16
|
||||
1 | 3 | 1 | 1 | 2 | 0 | 1 | 1 | 1 | 1
|
||||
2 | 1 | 0 | 0 | 1 | 0 | NULL | NULL | NULL | NULL
|
||||
[OK] simple GROUP BY <expression> (no subquery) [OK] simple implicit group-by
|
||||
StartEnd form A (first()/last()):
|
||||
0 | 6 | 5 | 4 | 1 | 1 | 1 | 5 | NULL | NULL | 00:00:00.5000000 | 1 | 7 | NULL | NULL | 00:00:10.4999990
|
||||
1 | 3 | 1 | 1 | 2 | 0 | 1 | 1 | NULL | NULL | 00:00:10.5000000 | 1 | 1 | NULL | NULL | 00:00:10.5000000
|
||||
2 | 1 | 0 | 0 | 1 | 0 | 0 | NULL | NULL | NULL | NULL | 0 | NULL | NULL | NULL | NULL
|
||||
StartEnd form B (row_number window): identical rows [OK] filter-in-subquery [OK] extra predicate in ON
|
||||
ActualTime (min): 0 ... | 2 | 00:00:01.5000000 1 ... | 1 | 00:00:10.5000000 2 ... | NULL | NULL
|
||||
ActualTime (max): 0 ... | 7 | 00:00:10.4999990 1 ... | 1 | 00:00:10.5000000 2 ... | NULL | NULL
|
||||
[OK] arg_min(ts, num_value) exists? 0 | 00:00:01.5 | 00:00:10.499999 1 | 00:00:10.5 | 00:00:10.5
|
||||
randomized cross-check (60k rows over 3 days (4 partitions), 2 tags, 7 min buckets, random statuses/kinds/tombstones):
|
||||
buckets=617 rowsInWindow=28423 mismatches=0; first()/last() 23 ms, window 21 ms, actual-time min+max 26 ms;
|
||||
arg_min/arg_max (one stmt) 11 ms, tie-break mismatches vs earliest-ts rule = 0
|
||||
```
|
||||
|
||||
Every hand-computed value matches: bucket 0 averages (5+2+2+7)/4 = 4, the tombstone is invisible, the sample one µs
|
||||
before the edge stays in bucket 0, the edge sample is in bucket 1, and both Empty rows count as bad whatever their
|
||||
status. The randomized cross-check compares counts, first/last usable times, sums and min/max actual times against a
|
||||
client-side computation. It used a µs-aligned start that is not on a second boundary, and integer-rounded values to
|
||||
force ties.
|
||||
|
||||
**Decision (the confirmed forms, with the SQL Server family split kept):**
|
||||
- Classified subquery `s`. Status is stored as the **unsigned** uint in a LONG, so usable = `status < 2147483648` and
|
||||
good = `status < 1073741824`, both also needing `kind <> 0`:
|
||||
```sql
|
||||
SELECT (cast(ts AS LONG) * 10 - $1) / $2 AS b, ts, kind, num_value, int_value, str_value,
|
||||
CASE WHEN status < 2147483648 AND kind <> 0 THEN 1 ELSE 0 END AS u,
|
||||
CASE WHEN status < 1073741824 AND kind <> 0 THEN 1 ELSE 0 END AS g
|
||||
FROM {p}sample WHERE tag = $3 AND ts >= $4 AND ts < $5 AND NOT deleted
|
||||
```
|
||||
`$1` = window start in **ticks since the Unix epoch** (`start.Ticks - DateTime.UnixEpoch.Ticks`, `Bigint`); `$2` =
|
||||
interval in ticks (`Bigint`); `$4`/`$5` = rounded window bounds. `cast(ts AS LONG)` is µs since 1970, so `*10` is
|
||||
ticks. That is exact because both sides are already µs-aligned (R6). At year 9999 `µs*10` ≈ 2.5e18 < 9.2e18, so it
|
||||
cannot overflow.
|
||||
- Common counts: `count() AS total_count, sum(u) AS usable_count, sum(g) AS good_count, count() - sum(u) AS bad_count,
|
||||
sum(CASE WHEN u = 1 AND kind IN (4, 5) THEN 1 ELSE 0 END) AS non_numeric_count`.
|
||||
- **Simple:** `SELECT b, <counts>, avg(CASE WHEN u = 1 THEN num_value END) AS avg_value, min(…), max(…), sum(…) FROM (<s>)
|
||||
GROUP BY b ORDER BY b`.
|
||||
- **StartEnd:** the window form, which is the same shape as SQL Server:
|
||||
`WITH s AS (<s>), st AS (SELECT b, <counts> FROM s GROUP BY b), f AS (SELECT b, kind, num_value, int_value,
|
||||
str_value, ts, row_number() OVER (PARTITION BY b ORDER BY ts) AS rf, row_number() OVER (PARTITION BY b ORDER BY ts
|
||||
DESC) AS rl FROM s WHERE u = 1) SELECT st.*, x.kind, x.num_value, x.int_value, x.str_value, x.ts, y.kind, …, y.ts
|
||||
FROM st LEFT JOIN (SELECT * FROM f WHERE rf = 1) x ON x.b = st.b LEFT JOIN (SELECT * FROM f WHERE rl = 1) y ON
|
||||
y.b = st.b ORDER BY st.b`. A bucket without a usable sample has NULL `ts` but **kind `0`** (BYTE has no NULL), so the
|
||||
evaluator must key "no first/last" on the NULL time.
|
||||
- **ActualTime:** `… r AS (SELECT b, num_value, ts, row_number() OVER (PARTITION BY b ORDER BY num_value ASC|DESC, ts
|
||||
ASC) AS rn FROM s WHERE u = 1 AND num_value IS NOT NULL) SELECT st.*, r.num_value, r.ts FROM st LEFT JOIN (SELECT *
|
||||
FROM r WHERE rn = 1) r ON r.b = st.b ORDER BY st.b`.
|
||||
- **One query per node holds for every family.** The S11 two-statement fallback is not needed. `first()/last()` and
|
||||
`arg_min/arg_max` also gave correct answers, but their tie order is not specified by an `ORDER BY`. The window form
|
||||
states the earliest-timestamp tie-break explicitly, so it is the one to use.
|
||||
- `SAMPLE BY` is not used.
|
||||
|
||||
## Q9 — AtTime neighbour lookup
|
||||
|
||||
200k rows (2 tags × 100k, ~1.5 s apart, 5 % tombstones). 102 request times, including one before the first sample,
|
||||
one exact hit and one after the last sample.
|
||||
|
||||
```
|
||||
form A SQL (one time): SELECT * FROM (SELECT 0 AS req, 0 AS dir, ts, kind, num_value, int_value, str_value, status FROM spike_q9
|
||||
WHERE tag = $1 AND ts <= $2 AND NOT deleted ORDER BY ts DESC LIMIT 1) UNION ALL SELECT * FROM (SELECT 0 AS req, 1 AS dir, ts, ...
|
||||
WHERE tag = $1 AND ts > $2 AND NOT deleted ORDER BY ts ASC LIMIT 1)
|
||||
form A chunk= 1: 61 ms for 102 times, rows=202, mismatches=0
|
||||
form A chunk= 10: 20 ms ... chunk= 25: 18 ms chunk= 50: 19 ms chunk= 60: 18 ms chunk= 63: 16 ms (all mismatches=0)
|
||||
form A chunk= 64: FAIL 'SQL model is too complex to evaluate' (128 UNION ALL branches)
|
||||
form A chunk=102: FAIL 'SQL model is too complex to evaluate' (204 UNION ALL branches)
|
||||
form B: 1 ms for 102 times (prev only), rows=102, mismatches=0 [ASOF JOIN of a long_sequence(1) UNION ALL request set]
|
||||
```
|
||||
|
||||
**Decision: form A, chunked `LIMIT 1` seeks, at most 50 request times (100 `UNION ALL` branches) per statement.**
|
||||
The exact limit is **126 `UNION ALL` branches (63 request times) per statement**. A re-measure with single-seek
|
||||
branches (review fix, below) showed 126 work and 127 fail. The form A runs above agree: 63 times (126 branches) work,
|
||||
and 64 times (128 branches) fail. The literal `req` index and `dir` (0 = previous-or-equal, 1 = next) columns are
|
||||
generated integers, not user input; `$1` is the tag and `$2..` the rounded times. It returns both neighbours in one
|
||||
shape and costs ~0.2 ms per time. ASOF JOIN is faster, but it only gives the previous neighbour, so a second
|
||||
seek form would still be needed for the next one. That does not justify two code paths at this scale.
|
||||
|
||||
## Q10 — long strings and SYMBOL capacity
|
||||
|
||||
```
|
||||
[OK] 400-char tag (SYMBOL) + 400-char str_value (VARCHAR)
|
||||
[OK] 5000-char str_value
|
||||
30,000 distinct SYMBOL values inserted in 723 ms
|
||||
count()=30002 | count_distinct(tag)=30002
|
||||
400-char round trip: tag equal=True str equal=True lengths=400,400 (includes non-ASCII αβγ…)
|
||||
tag | SYMBOL | symbolCached=True | symbolCapacity=16384
|
||||
point lookup by tag among 30k symbols: count=1 0 ms
|
||||
```
|
||||
|
||||
**Decision:** VARCHAR and SYMBOL take 400 characters (and VARCHAR takes 5,000). SYMBOL grows past its capacity hint
|
||||
(16,384) without error, so 30k tags is fine. No `CAPACITY` clause is needed.
|
||||
|
||||
## Q11 — existing-timestamp lookup for a batch
|
||||
|
||||
```
|
||||
[OK] range form: WHERE tag=$1 AND ts >= $2 AND ts <= $3 (tombstones included) -> 10 rows, incl. 00:00:08 deleted=True
|
||||
[OK] IN-list form: ts IN ($2..$11) -> 4 rows (00, 06, 12, 18)
|
||||
[OK] tombstone visible to range lookup -> 00:00:08 | True
|
||||
```
|
||||
|
||||
**Decision:** both work with parameters. **Live writer (§2.2 step 2): the range form**
|
||||
`SELECT ts FROM {p}sample WHERE tag = $1 AND ts >= $2 AND ts <= $3` (no `NOT deleted`, so tombstones count as present),
|
||||
then a client-side set test. **Updater (sparse item times): the IN-list form** `SELECT ts, deleted FROM {p}sample WHERE
|
||||
tag = $1 AND ts IN ($2, …)`, chunked at 1,000 times.
|
||||
|
||||
## Q12 — test cleanup and prefix listing
|
||||
|
||||
```
|
||||
LIKE 't12_%' (note: _ is a wildcard) -> t12_sample, t12xsample <- WRONG: matches t12xsample
|
||||
LIKE $1 with escaped _ ('t12\_%') -> t12_sample
|
||||
starts_with(table_name, $1) -> t12_sample
|
||||
left(table_name, 4) = $1 -> t12_sample
|
||||
[OK] DROP TABLE IF EXISTS t12_sample (exists) [OK] DROP TABLE IF EXISTS t12_sample (already gone)
|
||||
```
|
||||
|
||||
**Decision:** cleanup is `DROP TABLE IF EXISTS {name}`. List with **`SELECT table_name FROM tables() WHERE
|
||||
starts_with(table_name, $1)`**. Never use `LIKE 'prefix%'`, because `_` in `hist_` is a wildcard.
|
||||
|
||||
## Q13 — binding extreme DateTimes
|
||||
|
||||
Seeds: 1970-01-01, 1990-05-05T00:00:00.123456, 2026-03-01, 2200-01-01. "Round" is R6's rule: `MinValue` → `+10 ticks`,
|
||||
then `ticks - ticks % 10`.
|
||||
|
||||
```
|
||||
DateTime.MinValue raw -> server sees µs -9222425352054775808 (Npgsql sends -infinity)
|
||||
DateTime.MinValue Round -> 0001-01-01T00:00:00.0000010 (µs -62135596799999999)
|
||||
DateTime.MaxValue raw -> server sees µs -9222425352054775809 (Npgsql sends +infinity)
|
||||
DateTime.MaxValue Round -> 9999-12-31T23:59:59.9999990 (µs 253402300799999999)
|
||||
0001-01-01 raw -> µs -9222425352054775808 (== MinValue -> -infinity)
|
||||
1601-01-01 raw/Round -> 1601-01-01T00:00:00 (µs -11644473600000000)
|
||||
9999-12-31T23:59:59.9999999 raw -> µs -9222425352054775809 (== MaxValue -> +infinity)
|
||||
1969-12-31T23:59:59.9999995 raw -> 1970-01-01T00:00:00.0000000 (µs 0) Round -> 1969-12-31T23:59:59.9999990 (µs -1)
|
||||
1990-05-05T00:00:00.1234567 raw -> 1990-05-05T00:00:00.1234570 Round -> ...1234560
|
||||
1990-05-05T00:00:00.1234565 raw -> 1990-05-05T00:00:00.1234570 Round -> ...1234560
|
||||
2026-03-01T00:00:00.0000005 raw -> 2026-03-01T00:00:00.0000000 Round -> same
|
||||
range queries WHERE ts >= $1 AND ts < $2:
|
||||
Min..Max raw count=0 <- silently empty
|
||||
Min..Max Round count=4
|
||||
0001..9999.9999999 Round count=4
|
||||
1601..t0 Round count=2
|
||||
1990.1234565..1990.1234575 raw count=0 <- seed .123456 missed (bound arrived as .123457)
|
||||
1990.1234565..1990.1234575 Round count=1
|
||||
inserting extreme designated timestamps:
|
||||
[FAIL] insert ts=1969-12-31T23:59:59: PostgresException: 00000: designated timestamp before 1970-01-01 is not allowed
|
||||
[FAIL] insert ts=1601-01-01: ... [FAIL] insert ts=Min+10 ticks: ...
|
||||
[OK] insert ts=9999-12-31 Round -> 9999-12-31T23:59:59.9999990
|
||||
```
|
||||
|
||||
**Decision:** R6's `BindTimestamp` is **confirmed necessary and sufficient**.
|
||||
- A raw `DateTime.MinValue`/`MaxValue` (and anything equal to them) becomes ±infinity, and QuestDB stores that as a
|
||||
near-`long.MinValue` value. A `[Min, Max)` range then matches **nothing**, with no error.
|
||||
- A raw pre-2000 sub-µs value rounds **up**, which shifts a bound across a stored sample.
|
||||
- With Round, every bound behaves.
|
||||
- **New constraint:** a designated timestamp **before 1970-01-01** cannot be inserted. Reads with pre-1970 bounds work.
|
||||
See the spec §2.2 ruling candidate.
|
||||
|
||||
## DDL and DeleteRaw paging (extra)
|
||||
|
||||
```
|
||||
[OK] CREATE TABLE IF NOT EXISTS spike_hist_sample (...) TIMESTAMP(ts) PARTITION BY DAY TTL 30 DAYS WAL DEDUP UPSERT KEYS(ts, tag)
|
||||
[OK] CREATE TABLE IF NOT EXISTS spike_hist_event (...) TIMESTAMP(ts) PARTITION BY DAY TTL 30 DAYS WAL DEDUP UPSERT KEYS(ts, event_id)
|
||||
[OK] CREATE TABLE IF NOT EXISTS spike_hist_tag (...) TIMESTAMP(ts) PARTITION BY YEAR WAL DEDUP UPSERT KEYS(ts, name)
|
||||
[OK] re-run (idempotent) x3
|
||||
spike_hist_event | DAY | True | True | 30 | DAY
|
||||
spike_hist_sample | DAY | True | True | 30 | DAY
|
||||
spike_hist_tag | YEAR | True | True | 0 | HOUR
|
||||
hist_tag upsert-by-name at ts=1970-01-01 (tagA twice, tagB once):
|
||||
1970-01-01T00:00:00 | tagA | 4 | degF | second
|
||||
1970-01-01T00:00:00 | tagB | 1 | NULL | NULL
|
||||
DeleteRaw paging (page = 1000 rows): 2500 live rows + 1 outside range
|
||||
pages=3 tombstones=2500 in 103 ms
|
||||
deleted=False 1 deleted=True 2500
|
||||
```
|
||||
|
||||
DeleteRaw loop, as confirmed: `SELECT ts FROM {p}sample WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY
|
||||
ts LIMIT 1000` → one 1,000-row tombstone INSERT → the Q4 wait → continue from `last ts + 10 ticks`. The wait matters:
|
||||
without it the next page can re-read rows it just tombstoned.
|
||||
|
||||
## Review fixes (2026-10-11)
|
||||
|
||||
### Raw `LIMIT $n` with a bound parameter
|
||||
|
||||
`dotnet run -- limit` against 10.0.1, with 10 rows seeded in `[t0, t0+10 s)`. The SQL was `SELECT ts, num_value FROM
|
||||
spike_lim WHERE tag = $1 AND ts >= $2 AND ts < $3 AND NOT deleted ORDER BY ts [DESC] LIMIT $4`:
|
||||
|
||||
```
|
||||
rows=3 first=0 last=2 [OK] ASC LIMIT $4 (Integer 3)
|
||||
rows=3 first=9 last=7 [OK] DESC LIMIT $4 (Integer 3)
|
||||
rows=3 first=0 last=2 [OK] ASC LIMIT $4 (Bigint 3)
|
||||
rows=3 first=9 last=7 [OK] DESC LIMIT $4 (Bigint 3)
|
||||
rows=10 first=0 last=9 [OK] ASC LIMIT $4 (Integer int.MaxValue)
|
||||
rows=10 first=9 last=0 [OK] DESC LIMIT $4 (Integer int.MaxValue)
|
||||
rows=10 first=0 last=9 [OK] ASC LIMIT $4 (Bigint long.MaxValue)
|
||||
rows=10 first=9 last=0 [OK] DESC LIMIT $4 (Bigint long.MaxValue)
|
||||
rows=0 first=NULL last=NULL [OK] ASC LIMIT $4 (Integer 0)
|
||||
rows=0 first=NULL last=NULL [OK] DESC LIMIT $4 (Integer 0)
|
||||
```
|
||||
|
||||
**Decision:** a bound `LIMIT $n` works, with `NpgsqlDbType.Integer` (Int32) or `Bigint` (Int64). Use `Integer`, since
|
||||
`top` is an `int`. R8's unbounded value `int.MaxValue` returns every row. `LIMIT 0` returns nothing, but R8 never sends
|
||||
it. The inlined-literal fallback is not needed.
|
||||
|
||||
### Exact `UNION ALL` branch limit
|
||||
|
||||
`dotnet run -- union`. Each branch is `SELECT * FROM (SELECT k AS req, ts FROM spike_un WHERE tag = $1 AND ts <= $2 AND
|
||||
NOT deleted ORDER BY ts DESC LIMIT 1)`:
|
||||
|
||||
```
|
||||
[OK] 100 branches [OK] 120 branches [OK] 125 branches [OK] 126 branches
|
||||
[FAIL] 127 branches: PostgresException: 00000: SQL model is too complex to evaluate
|
||||
[FAIL] 128 branches / 129 / 130: same
|
||||
```
|
||||
|
||||
**Decision:** the limit is **126 branches, which is 63 AtTime request times** (two branches per time). The chosen chunk
|
||||
of **50 times (100 branches)** is unchanged.
|
||||
|
||||
## Rulings candidates raised by the spike
|
||||
|
||||
1. **Pre-1970 timestamps cannot be stored** (Q13). QuestDB rejects a designated timestamp before 1970-01-01, and the
|
||||
whole multi-row INSERT fails. Proposed:
|
||||
- the live writer drops such samples and logs one Warning per batch. Retrying cannot succeed, and sending the
|
||||
sample would fail the tag's whole group into #538 quarantine;
|
||||
- HistoryUpdate Insert/Replace/Update items before 1970 return `BadOutOfRange`;
|
||||
- DeleteAtTime/DeleteRaw find nothing there and return `BadNoEntryExists`;
|
||||
- reads are unaffected.
|
||||
Conformance tests that seed pre-1970 data are skipped for QuestDB.
|
||||
2. **`long.MinValue` is QuestDB's LONG NULL** (sentinel check). An Int64 sample of exactly `long.MinValue` would read
|
||||
back `int_value` NULL, and `FromColumns` would then serve **0**. Proposed: the QuestDB read path repairs it. When
|
||||
`kind = Int64`, `int_value IS NULL` and `num_value = -9.223372036854775808E18` (exact in a double), serve
|
||||
`long.MinValue`. That is a local fix-up before the shared `FromColumns`, with no write-side change.
|
||||
3. **TTL metadata is unit-normalised and applied asynchronously** (Q5). The bootstrapper compares in hours and does not
|
||||
re-read straight after an `ALTER`.
|
||||
@@ -0,0 +1,70 @@
|
||||
# Issue batch #667–#673 — design (2026-10-11)
|
||||
|
||||
All seven open issues ≥ #667 are testable with available infra (docker-dev rig, SQL Server, opc-plc fixture on
|
||||
10.100.0.35). Base: master `ee67b637`. Recommended choices taken throughout (user-approved).
|
||||
|
||||
## Lanes
|
||||
|
||||
| Lane | Issue | Model | Wave |
|
||||
|---|---|---|---|
|
||||
| L1 | #673 FOCAS flake | sonnet | 1 |
|
||||
| L2 | #670 OpcUaClient session loss | opus | 1 |
|
||||
| L3 | #672 crash-failover capture gap | opus | 1 |
|
||||
| L4 | #669 event HistoryRead continuation points | sonnet | 1 |
|
||||
| L5 | #667 SqlServer retention/purge | sonnet | 1 |
|
||||
| L6 | #671 time-weighted Part 13 aggregates | opus | 1 |
|
||||
| L7 | #668 Part 11 modified values | opus | 2 (after L4 + L5 merge) |
|
||||
|
||||
Each lane: isolated worktree reset to master, TDD, independent opus review, merge, full suite, docker-dev live gate.
|
||||
|
||||
## #673
|
||||
Root cause: `FocasDriver.FixedTreeLoopAsync` publishes `LastStatus` before `FixedTreeCache` / `LastFixedPollUtc`;
|
||||
the test waits on `LastStatus` only, and an authored `FIXED:Status/*` read returns `BadCommunicationError` while
|
||||
`LastFixedPollUtc` is null. Fix: test waits on all three; driver writes `LastStatus` last via `Volatile.Write`.
|
||||
|
||||
## #670
|
||||
Root causes: bad keep-alive never touches `_health` nor publishes; `LastSuccessfulRead` never advances on
|
||||
notifications/keep-alives; the SDK `SessionReconnectHandler` retries forever (exhausted path unreachable) and on a
|
||||
session recreate clones subscriptions the driver's dictionaries no longer track.
|
||||
Fix (Option B): bad keep-alive ⇒ `Degraded` (stamp preserved) + one `BadNoCommunication` publish per subscribed ref
|
||||
per outage (value retained) + quality-only alarm update; good keep-alive / notifications advance
|
||||
`LastSuccessfulRead`; a self-healed short outage re-reads and republishes Good (beats the (value,status) dedup);
|
||||
sustained loss is recovered by the #527 watchdog (shutdown + init + resubscribe). Delete `SessionReconnectHandler`
|
||||
wiring, `OnReconnectComplete`, `TryRearmReconnect`, the `reconnect_exhausted` counter. Wire SDK telemetry logging
|
||||
at Warning. Correct `docs/DriverLifecycle.md` survey.
|
||||
|
||||
## #672
|
||||
Standby opens the historization capture gate when its recording Primary is cluster-UNREACHABLE (local
|
||||
`ClusterEvent.ReachabilityEvent`, recomputed from the cached redundancy snapshot in `DriverHostActor`). Alarm drain
|
||||
gate and data-plane Primary gate unchanged (falsification tests). Knob
|
||||
`ContinuousHistorization:CaptureWhilePrimaryUnreachable` (default true). No backend capability flag. Gap shrinks to
|
||||
the failure-detector window (~10–12 s). Documented: drivers without a source timestamp (Modbus) duplicate rows in
|
||||
the overlap window.
|
||||
|
||||
## #669
|
||||
Event HistoryRead issues/resumes continuation points: cursor = last event time + trailing tie count; oversized tie
|
||||
cluster over-fetch via `[T, T+resolution)` bounded by `MaxTieClusterOverfetch`; generic `SliceTieCluster`; store
|
||||
state widened to carry an event cursor; unknown CP ⇒ `BadContinuationPointInvalid`. Reverse event reads out of
|
||||
scope. Ties exact on SqlServer/QuestDB, best-effort on Gateway (documented). Adds an over-the-wire event HistoryRead
|
||||
test.
|
||||
|
||||
## #667
|
||||
`ServerHistorian:SqlServer:{RetentionDays (null=forever), PurgeInterval=1h, PurgeBatchSize=4000}`; optional
|
||||
`IHistorianBackendProvider.CreateRetention`; `HistorianRetentionService` gated on `IHistorizationRoleView`
|
||||
plus `sp_getapplock` (timeout 0); per-tag batched `DELETE TOP(n)` on the clustered key, events by time; cutoff on
|
||||
source time; metric `otopcua.historian.retention.purged{table}`. QuestDB: no change (TTL). Partition switching
|
||||
deferred/documented.
|
||||
|
||||
## #671
|
||||
New time-weighted family: TimeAverage, TimeAverage2, Interpolative, Total (now the Part 13 time-weighted integral,
|
||||
matching Gateway — client-visible, release-noted), Total2. Bounding values outside the window + `LEAD()` in-bucket
|
||||
areas in SQL; C# splits bucket-crossing segments. Sloped for numeric, stepped for Boolean. Server-default
|
||||
AggregateConfiguration (client's ignored). QuestDB mirrored after a lag/lead support spike. Hand-computed literals
|
||||
in the shared conformance suite.
|
||||
|
||||
## #668
|
||||
`hist.SampleModified` (schema v2), prior values via `OUTPUT deleted.* INTO` in the same statement;
|
||||
`HistoryUpdateContext(Principal, ApiKeyId)` threaded into updater methods; Insert operations logged (empty prior);
|
||||
retention purges never logged but #667 purges `SampleModified`; `ReadModifiedAsync` serves `IsReadModified=true`
|
||||
with continuation points, authorized by the HistoryRead gate. QuestDB: append-only `sample_modified` written before
|
||||
the change (orphan-audit-row-on-crash, documented deviation).
|
||||
@@ -0,0 +1,70 @@
|
||||
# Follow-up batch #674–#678 — design (2026-10-11)
|
||||
|
||||
Follow-ups filed after the #667–#673 batch. Base: master `bc03adb1`. User-approved; recommended choices throughout.
|
||||
Investigation found three new defects (client endpoint port not rewritten; role overlays never load on
|
||||
cluster-role nodes; Gateway Count/Average mode mapping) — the first two are fixed here, the third is filed.
|
||||
|
||||
| Lane | Scope | Model |
|
||||
|---|---|---|
|
||||
| L1 | #674 HistoryUpdate posture | opus |
|
||||
| L2 | #675 reverse event paging | opus |
|
||||
| L3 | #677 aggregates | opus |
|
||||
| L4 | client (#677.5, #678.5/.6, #674.6) | sonnet |
|
||||
| L5 | #676 OpcUaClient | opus |
|
||||
| L6 | #678.1–.4 + role-overlay bug | sonnet |
|
||||
|
||||
## L1 — #674
|
||||
- **674.1:** spike on QuestDB 10.0.1 showed TTL is judged against `min(maxTs, wall clock)` (`cairo.ttl.use.wall.clock=true`
|
||||
default) — a 2066 row evicted nothing. Add a QuestDB integration test pinning that + a docs note. HistoryUpdate refuses a
|
||||
SourceTimestamp beyond `now + ServerHistorian:HistoryUpdateMaxFutureSkew` (default 1 day) with `BadOutOfRange`; the
|
||||
recorder is not bounded.
|
||||
- **674.2:** SqlServer base schema requirement back to v1; `SupportsModifiedTrail` (v ≥ 2). On v1: modified reads and
|
||||
HistoryUpdate ⇒ `BadHistoryOperationUnsupported` (nothing sent), retention skips `SampleModified`, one Warning per boot.
|
||||
- **674.3:** HistoryUpdate Primary-gated (off-Primary ⇒ `BadInvalidState`, same shape as `NativeAlarmAckPrimaryGate`).
|
||||
Retry duplicates on QuestDB documented; no DEDUP key.
|
||||
- 674.4/674.5 documented only.
|
||||
|
||||
## L2 — #675
|
||||
- Reverse event reads: `reverse` flag on `IHistorianDataSource.ReadEventsAsync`; SqlServer/QuestDB order DESC and take
|
||||
the top N; direction-aware cursor (reuse raw `SliceTieCluster(reverse:)`); Gateway ⇒ `BadHistoryOperationUnsupported`
|
||||
for reverse.
|
||||
- End-only event read ⇒ backward (mirrors raw).
|
||||
- Offline test pinning `DateTime.MaxValue` through the Gateway proto/FILETIME encoding; live AVEVA leg stays open (VPN).
|
||||
|
||||
## L3 — #677
|
||||
- Reverse processed reads per SDK/Part 13: buckets anchored at the later (start) time stepping back, partial bucket
|
||||
earliest, bucket timestamp = later edge. Parity test vs SDK `AggregateCalculator`. Client-visible, release-noted.
|
||||
- Partial bit also when the data starts inside a bucket (no bounding prior).
|
||||
- `AggregateConfiguration`: `UseServerCapabilitiesDefaults` or values equal to the server defaults ⇒ served; otherwise
|
||||
per-node `BadAggregateConfigurationRejected`.
|
||||
- Gateway provisioner sends `IntegralDivisor = 1` (proto3 default 0). Live magnitude check stays open (VPN).
|
||||
|
||||
## L4 — client
|
||||
- `EndpointSelector.SelectBest` rewrites host **and** port of the advertised endpoint (fixes rig `:4841` cert error).
|
||||
- Client aggregate enum/mapper/CLI/UI gain TimeAverage, TimeAverage2, Total, Total2, Interpolative.
|
||||
- History read refusal ⇒ one-line `CommandException`, no stack trace.
|
||||
- `NodeManagerHistoryReadTests` non-historized case sets `IsReadModified=false`.
|
||||
|
||||
## L5 — #676
|
||||
- Driver endpoint host+port rewrite; in-process TCP-stall proxy wire test (in-place blip, same session, Good republish,
|
||||
no regression).
|
||||
- Remove dead `ReconnectPeriod` (options, AdminUI form, docs, tests); stored JSON with it still parses.
|
||||
- New opt-in `IDriverConnectionLossSource` (Core.Abstractions): raised once per session by OpcUaClient only on a
|
||||
definitive loss (`BadSessionIdInvalid`/`BadSessionClosed`/`BadSessionNotActivated`/subscription lost);
|
||||
`DriverInstanceActor` (attach PreStart, detach PostStop) turns it into `DisconnectObserved` only while Connected.
|
||||
Ambiguous codes keep going through the #527 watchdog.
|
||||
- Information-level per-recovery replay-drop summary. Equal-timestamp drop documented.
|
||||
|
||||
## L6 — small
|
||||
- Retention: mid-command cancel test + `ClearPool` fallback test with falsification control.
|
||||
- #672 tests: initial-state unreachable path; strengthen `Self_unreachable_is_ignored`; edge-triggered Debug log.
|
||||
- FOCAS `WireBackendTests`: wait for motor speeds too; `WaitFor` fails on timeout.
|
||||
- QuestDB TTL-cleared log wording; `docker-dev/docker-compose.debug.yml` Debug recipe.
|
||||
- Role overlay suffix built from plain roles only — after auditing each `appsettings.<role>.json` and stripping anything
|
||||
that would change production behaviour on activation.
|
||||
|
||||
## Live gates (docker-dev)
|
||||
1. opc-plc `docker restart` recovers in seconds (was ~100 s); pause-blip re-check.
|
||||
2. Client.CLI connects to `:4841`–`:4845`.
|
||||
3. HistoryUpdate refused on the Secondary, accepted on the Primary; v1-schema DB keeps historizing.
|
||||
4. Reverse event paging and reverse processed reads over the wire, SqlServer + QuestDB.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Alarm delivery identity results — lmxopcua #705
|
||||
|
||||
Alarm events now carry a nonzero delivery GUID created once for direct delivery and preserved across retries and record copies. It is excluded from queue JSON and business-field equality/hash. Queued delivery overrides it with a frozen versioned UUID encoding derived only from the persisted queue row key, preserving existing payload JSON, fingerprints, schema, deduplication and AlarmId. The Gateway mapper sends that GUID and rejects explicitly empty identities. SQL Server and QuestDB identity logic is unchanged.
|
||||
|
||||
Coverage includes literal historical JSON/fingerprint/GUID vectors; ten-field equality and hash behavior; direct copies/retries; queue retry/reopen/payload edits/dead-letter revival; identical-input convergence; distinct same-time business events; real replicated queue convergence; and legacy migration. Reconstructed direct objects receive new identities; durable reconstruction uses the queue. Completely identical business payloads remain intentionally deduplicated.
|
||||
|
||||
TDD: direct wire regression failed on an empty ID; queued retry/reopen regression failed on changing IDs. Final affected runs: Core51/51, Host migration/convergence16/16, Gateway453passed/6live skipped/0failed. Full offline solution:14,635passed/611skipped/0failed across62suites, exit0. Independent final review and diffcheck clear. Fixture configuration variables were removed before all runs.
|
||||
|
||||
No live event writes, deployed sender rollout or historical row rewrite occurred. The live round-trip test now requires exact delivery GUID readback; actual persistence remains the open Gateway #28 gate, alongside its separate #39 timestamp-offset provider failure. Upstream History controls #29 and lmxopcua #694/#700 remain open.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Gateway tag-presence concurrency coverage — 2026-10-13
|
||||
|
||||
Approved tests-only fix for #701. Added 15 deterministic cases for overlapping positive/negative cold lookups, expiry refreshes, capacity crossing, successful-result isolation from faults/cancellation, and cancellation after asynchronous completion. Fake presence calls support an async hook and atomic counter. Production cache is unchanged; no single-flight promise.
|
||||
|
||||
The synthetic 16-caller cold/expiry bursts produced 16 catalog RPCs. Assertions permit one through16; this records amplification without making it a requirement or a live-cost claim. Newclass15/15 and Gateway449/449 passed. Independent review found no actionable defects; diff check clean.
|
||||
|
||||
First full solution run:14,614passed/609skipped/1failed across62suites. Existing restart/reconnect wire test failed, then isolated retry passed1/1. Full repeat with detailed console logging passed14,615/609skip/0fail across62suites, exit0, including OPC UA integration122/122 and host integration268passed/12skip. #702 tracks the unresolved intermittent failure; it is not closed merely because the repeat passed.
|
||||
|
||||
Both full runs excluded configured live fixture variables from the environment; environment-gated tests skipped. Existing NU1510/xUnit duplicate-InlineData warnings remain pre-existing. No production behavior or deployment changed by this patch.
|
||||
@@ -0,0 +1,36 @@
|
||||
# Missing-tag and Gateway performance fixes — 2026-10-13
|
||||
|
||||
Follow-up to pushed `97668d26` / PR #699. Implements #653 and the remaining #696 items 2/3 under the approved bounded design.
|
||||
|
||||
## Changes
|
||||
|
||||
- **#653:** Raw, Processed and AtTime check tag presence before history RPCs. The bounded 4,096-name cache uses a 5-second negative and 60-second positive lifetime. Only explicit absence or NotFound is cached as missing; failures and cancellation propagate. Raw is empty; AtTime and Processed retain requested/bucket timestamps with null/BadNoData. Unsupported aggregate validation runs first.
|
||||
- **Live compatibility adjustment:** the deployed gateway misparses missing `GetTagMetadata` replies as `Internal` (`InvalidDataException: GetTagInfo response ended unexpectedly`). The adapter therefore checks the filtered tag catalog and compares exact names, fully draining the stream before declaring presence/absence. It never turns the parser error into a missing tag. Wildcard matches alone do not prove presence, and a trailing stream fault cannot enter the cache.
|
||||
- **#696 item 2:** a truncated remainder probe seeds strict-ceiling reverse discovery with its observed rows/density; gap searches use the available discovery budget. Complete segments alone contribute to the newest-first page. Regression: a 7,200-row stopped 1 Hz tag, ceiling 1,000, requested latest three one day later, previously 17 raw calls; now meets a maximum of 12 with exactly 7199/7198/7197 returned. Ordinary page sizing and event discovery behavior are unchanged.
|
||||
- **#696 item 3:** per-sample aggregates reuse a ceiling refusal for 30 seconds instead of immediately re-probing on every full stream. The next incomplete aggregate read after expiry notices a raised ceiling; a lowered ceiling is re-learned immediately. Raw and AtTime re-probes remain immediate. During the cooldown an oversized aggregate remains BadTooManyOperations, never a truncated result.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
New regressions failed before fixes: all initial 12 cases failed (missing preflight/caching, 17 discovery reads, duplicate aggregate refusal). The four catalog wire cases failed with the metadata adapter and pass with the catalog adapter. Final Gateway suite: 434 passed, six environment-gated skips.
|
||||
|
||||
Independent read-only review found no important defects. Concurrent cold requests can duplicate a metadata/catalog check; cache access and capacity mutations are locked. No single-flight guarantee was introduced. Catalog RPC contract streams all matches; the upstream SDK pages the native tag-query results, with no configured browse result ceiling.
|
||||
|
||||
## Live #653 confirmation
|
||||
|
||||
Measured the updated GatewayHistorianDataSource through the real adapter and local gateway against the existing remote AVEVA connection. Temporary historian:read key had a two-hour expiry and was revoked after the run. No historian writes/provisioning and no gateway deployment changes.
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`.
|
||||
|
||||
| Operation, five consecutive runs | Elapsed milliseconds | Result |
|
||||
|---|---|---|
|
||||
| Raw | 228, 0, 0, 0, 0 | empty |
|
||||
| Count processed | 2, 0, 0, 0, 0 | six null/BadNoData buckets |
|
||||
| AtTime | 1, 0, 0, 0, 0 | null/BadNoData at requested time |
|
||||
|
||||
These used one data source/cache: the first Raw call is cold, subsequent calls are cached. The prior direct-client measurement took ~30.6 seconds for missing-tag Raw/Aggregate and ~61.3 seconds for AtTime. Direct client RPCs are not modified; the fast-fail protection is in OtOpcUa.
|
||||
|
||||
**Control limitation:** SysTimeSec catalog presence succeeded, but its Raw RPC twice timed out upstream with DeadlineExceeded at ~30 seconds. The data source propagated IOException/communication failure rather than returning no data. A fresh control using the original package client (no data-source presence checks) also timed out with DeadlineExceeded after 30,656 ms, confirming this timeout is independent of the new checks. The earlier direct-client control had succeeded. This run therefore verifies missing-tag fast-fail and failure propagation, not healthy-tag live history behavior. No end-to-end deployed OPC UA claim is made.
|
||||
|
||||
#694 remains open for native aggregate rerun after historiangw#27. No sibling gateway code changed.
|
||||
|
||||
Full final-tree solution: 14,600 passed, 609 skipped, 0 failed across 62 suites. `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Includes 122 passing OPC UA integration tests and 268 passing host integration tests (12 host skips). Environment-gated suites are included in the skip count.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Historian follow-up results — 2026-10-13
|
||||
|
||||
Scope: approved bounded fixes for #695–#698, plus live measurement of #653. Base: `66a2b0c3`.
|
||||
|
||||
- #695: spend the remaining learned-ceiling room on crowding reads; reduce crowded reverse-read windows using the observed pre-window count. Regression cases cover ceilings 1,709 and 1,369, and reverse paging through a crowded millisecond.
|
||||
- #696 item 1: probe the future interval before starting the prior-sample search near now. Preserve stored future samples, requested timestamps and Uncertain status. Dense future probes still use the existing bounded walk. Items 2 and 3 remain open (stopped-tag strict-ceiling cost and aggregate ceiling re-probes).
|
||||
- #697: actual Gateway, SQL Server and QuestDB transport failures become `BadNoCommunication` on every HistoryRead arm. Cancellation remains `BadTimeout`; unsupported operations retain `BadHistoryOperationUnsupported`. Tests include real refused database connections, repeated reads and nested gRPC failures.
|
||||
- #698: normalize Raw, Modified and AtTime scalar values to the node DataType. Conversion failure gives per-sample null/`BadTypeMismatch`; arrays, audit metadata and timestamps survive. Processed results retain aggregate-specific types.
|
||||
- #694 item 3 remains blocked on historiangw#27. No upstream code was changed.
|
||||
|
||||
## Live #653 measurement
|
||||
|
||||
Five sequential calls per operation and tag, using client 0.3.0 against the running local gateway (`localhost:5221`) with its existing remote AVEVA connection. Commissioned a two-hour `historian:read` key, then revoked it after measurement. No historian writes or provisioning performed.
|
||||
|
||||
| Operation | SysTimeSec, elapsed ms | Nonexistent tag, elapsed ms |
|
||||
|---|---|---|
|
||||
| Metadata | 299, 85, 81, 82, 48 (success) | 89, 53, 41, 77, 76 (gateway exception) |
|
||||
| Raw | 143, 99, 145, 133, 159 (success) | 30260, 30634, 30675, 30754, 30628 (gateway exception) |
|
||||
| Aggregate | 271, 275, 309, 249, 304 (success) | 30564, 30605, 30583, 30575, 30806 (gateway exception) |
|
||||
| AtTime | 140, 88, 89, 127, 92 (success) | 61074, 61330, 61601, 61249, 61271 (gateway exception) |
|
||||
|
||||
Nonexistent tag: `OtOpcUa.Issue653.Nonexistent.20261013`. The delay is still present. Recommend a metadata pre-check with a short negative cache, caching only explicit missing-tag outcomes and never authentication, communication or arbitrary errors. This is follow-up implementation; #653 remains open. Direct gateway AtTime above differs from OtOpcUa's raw-based AtTime evaluator.
|
||||
|
||||
## Consumer and validation notes
|
||||
|
||||
ScadaBridge source and tests contain no HistoryRead consumer; no consumer change required. Independent review found a future-stored-sample edge case, which was fixed and covered before final verification. No deployment or live end-to-end OPC UA verification of these new fixes is claimed; the live run measures #653 through the gateway client.
|
||||
|
||||
Full solution: 14,581 passed, 609 skipped, 0 failed across 62 test suites. Command: `dotnet test ZB.MOM.WW.OtOpcUa.slnx --verbosity quiet`, exit 0. Skips include environment-gated integration suites.
|
||||
@@ -0,0 +1,9 @@
|
||||
# Restart loss reason test correction — #702
|
||||
|
||||
The full integration reproduction failed at the restart wire test reason assertion: it required BadServerHalted but correctly received BadSessionIdInvalid. Production already supports that definitive loss reason.
|
||||
|
||||
The test now accepts an explicit semantic set: four direct definitive session statuses and two sustained closed-channel statuses. All BadNoCommunication, retained-value, unhealthy-state, frozen-read-stamp, one-notification and explicit reinitialization/recovery assertions remain unchanged. Only test expectation/comments changed; production unchanged. Existing deterministic driver tests independently enforce direct-status ordering/count and the sustained-channel threshold.
|
||||
|
||||
Verification: isolated restart1/1, deterministic status routes30/30, full OPC UA integration122/122. Full offline solution: 14,613 passed / 611 skipped / 0 failed, exit0, 61 reported suite summaries. Host integration266 passed /14 skipped; unavailable environment/fixture tests remain skips. Independent review found no important defects; git diff --check passed. Actual prior red run121pass/1fail, detailed console/TRX confirms the exact reason mismatch.
|
||||
|
||||
Logs/TRX retained in the Codex thread workspace under work/issue702-*. No Gateway or deployment behavior changed by this test correction. Separate live gates remain open.
|
||||
+20
-2
@@ -416,7 +416,9 @@ write and each Part 9 method call first asks `DataPlaneAuthorizer.AuthorizeKeyUs
|
||||
`AclGate: denied … key … is no longer enabled or has expired`), checked *before* the "no gateway ⇒
|
||||
`BadNotWritable`" availability answer. Things to know:
|
||||
|
||||
- **Only these two gates.** Read, Browse, Subscribe and HistoryRead stay ungated with the flag off —
|
||||
- **Only these two gates — plus HistoryUpdate.** (HistoryUpdate, added with the pluggable historian, is a third gated
|
||||
operation in both modes; see [HistoryUpdate](#historyupdate-historian-writes). It re-checks the key the same way.)
|
||||
Read, Browse, Subscribe and HistoryRead stay ungated with the flag off —
|
||||
that is the flag-off posture, and an expired key's session can still read and keep its subscriptions.
|
||||
Turn `Security:Acl:Enabled` on to have expiry reach those too.
|
||||
- **Anonymous is unaffected.** It carries no key id, so there is nothing to expire; its roles stay what
|
||||
@@ -484,6 +486,7 @@ the seams consult. Same dark-switch discipline as `MeshTransport`, `ConfigSource
|
||||
| Alarm Confirm | role `AlarmAck` | `AlarmConfirm` |
|
||||
| Alarm OneShotShelve / TimedShelve / Unshelve | role `AlarmAck` | `AlarmShelve` |
|
||||
| HistoryRead (all four variants) | **nothing** | `HistoryRead` on each node's scope, decided **per node in the batch** |
|
||||
| HistoryUpdate (insert / replace / update / delete) | role `HistoryUpdate` (from a `Cluster`-scoped grant) + key re-check | `HistoryUpdate` on that node's scope, decided **per node in the batch** |
|
||||
| Read (every attribute, not just Value) | nothing | `Read` on that node's scope, decided per node in the batch |
|
||||
| Browse / TranslateBrowsePath | nothing | `Browse` on the source node, **and** on each returned reference |
|
||||
| CreateMonitoredItems / SubscribeToEvents | nothing | `Subscribe` on that node's scope, decided per item |
|
||||
@@ -872,7 +875,22 @@ public enum NodePermissions : int
|
||||
|
||||
The three Write tiers map to Galaxy's v1 `SecurityClassification` — `FreeAccess`/`Operate` → `WriteOperate`, `Tune` → `WriteTune`, `Configure` → `WriteConfigure`. `SecuredWrite` / `VerifiedWrite` / `ViewOnly` classifications remain read-only from OPC UA regardless of grant.
|
||||
|
||||
`HistoryUpdate` (bit 12) is a **separate** permission from `HistoryRead` so a read-only grant cannot also authorize OPC UA HistoryUpdate (annotation / insert / delete) operations. `TriePermissionEvaluator` maps `OpcUaOperation.HistoryUpdate` to this bit, closing the read⇒update hole that existed when the only history permission was `HistoryRead`. Note: `HistoryUpdate` is **not** included in any composite bundle (`ReadOnly` / `Operator` / `Engineer` / `Admin`) because the HistoryUpdate service surface (the actual insert/replace/delete backend RPC) is not yet implemented — a client calling HistoryUpdate still receives the SDK's default reject regardless of the grant.
|
||||
`HistoryUpdate` (bit 12) is a **separate** permission from `HistoryRead` so a read-only grant cannot also authorize OPC UA HistoryUpdate operations. `TriePermissionEvaluator` maps `OpcUaOperation.HistoryUpdate` to this bit. It is **not** included in any composite bundle (`ReadOnly` / `Operator` / `Engineer` / `Admin`) — it must be granted explicitly, and it is offered as its own checkbox in the AdminUI ACL editor (`AclEdit.razor`). The HistoryUpdate service itself is now implemented (SQL Server historian backend only) — see the next section.
|
||||
|
||||
### HistoryUpdate (historian writes)
|
||||
|
||||
HistoryUpdate rewrites recorded history, so unlike HistoryRead it is gated **whatever `Security:Acl:Enabled` says**, and `WriteOperate` (live writes) never implies it.
|
||||
|
||||
- **Flag off (default):** the session's API key is re-checked on the call (`DataPlaneAuthorizer.AuthorizeKeyUsable`, Gitea #563 — an expired or disabled key loses its activation-time roles), then the session must carry the **`HistoryUpdate` role** (`OpcUaDataPlaneRoles.HistoryUpdate`). `ApiKeyInterimRoleMapper` confers it **only from a `Cluster`-scoped grant** carrying `NodePermissions.HistoryUpdate` — narrower-scoped grants confer nothing, for the same reason as `WriteOperate` (honouring a flag without honouring its scope would widen it). A key holding only `WriteOperate` is denied `BadUserAccessDenied`.
|
||||
- **Flag on:** `DataPlaneAuthorizer.Authorize(identity, OpcUaOperation.HistoryUpdate, nodeId, realm)` decides per node and **replaces** the role check (never chained) — a session with no roles succeeds on the strength of a scoped `HistoryUpdate` grant, and a `WriteOperate` grant does not help.
|
||||
- **Availability is separate from authorization.** A node accepts HistoryUpdate only if it is historized **and** the selected historian backend has an updater when the node materialises (the `HistoryWrite` AccessLevel bit). `Backend=Gateway` has none, so it answers `BadHistoryOperationUnsupported` for every node whatever the grant. Operations, result codes and unsupported cases: [`docs/Historian.md` §HistoryUpdate](Historian.md#historyupdate).
|
||||
- **Primary-only, after authorization (Gitea #674).** An authorized HistoryUpdate on a node that is not the redundancy Primary is refused `BadInvalidState` (the native-alarm-ack verdict, #597); an unauthorized one is told `BadUserAccessDenied` first, so the refusal never reveals the node's role to a caller who may not update.
|
||||
- **Every applied change is attributed (Part 11 modified-value history, Gitea #668).** The session's principal (the API key's
|
||||
`NodeAcl` principal, or the reserved `anonymous`) and key id are passed to the backend with every HistoryUpdate and recorded in the
|
||||
modification trail next to the prior value; a modified-value HistoryRead (`IsReadModified = true`) returns it as
|
||||
`ModificationInfo.UserName`. Reading the trail is gated by the **same `HistoryRead` decision** as a raw read — there is no separate
|
||||
permission — so with `Security:Acl:Enabled` off it is as ungated as any HistoryRead, and anyone who may read a node's history can see
|
||||
who rewrote it and what it held before. See [`docs/Historian.md` §Modified-value history](Historian.md#modified-value-history-part-11).
|
||||
|
||||
### Evaluator — `PermissionTrie`
|
||||
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@ Per-driver-instance child. State machine:
|
||||
|
||||
`ShouldStub(driverType, roles)` returns `true` for `"Galaxy" | "Historian.Wonderware"` on non-Windows; the actor goes straight to `Stubbed` and returns deterministic success without touching real hardware. Wiring this into the DriverHost child-spawn path is follow-up F20 (folds into F7).
|
||||
|
||||
Engine wiring (subscription publishing, ApplyDelta diff, bad-quality-on-disconnect, write path, supervisor backoff) is stubbed — tracked as F7. Tests exercise message contracts, not engine behaviour.
|
||||
Engine wiring (subscription publishing, config-change diff, bad-quality-on-disconnect, write path, supervisor backoff) is stubbed — tracked as F7. Tests exercise message contracts, not engine behaviour.
|
||||
|
||||
## VirtualTagActor / ScriptedAlarmActor
|
||||
|
||||
|
||||
@@ -815,7 +815,6 @@ Already used by Client.Shared in the existing codebase.
|
||||
| `Password` | string? | — | For Username auth |
|
||||
| `SessionTimeoutMs` | int | 120000 | Server-negotiated session timeout |
|
||||
| `KeepAliveIntervalMs` | int | 5000 | Session keep-alive interval |
|
||||
| `ReconnectPeriodMs` | int | 5000 | Initial reconnect delay |
|
||||
| `BrowseStrategy` | enum | Full | `Full` (browse all at startup), `Lazy`, `Hybrid` |
|
||||
| `BrowseRoot` | string? | — | Optional root NodeId to mirror (default: ObjectsFolder) |
|
||||
| `AutoAcceptCertificates` | bool | false | Accept untrusted server certificates |
|
||||
|
||||
@@ -47,7 +47,7 @@ public class HistoryReadCommand : CommandBase
|
||||
/// <summary>
|
||||
/// Gets the optional aggregate name used when the operator wants processed history instead of raw samples.
|
||||
/// </summary>
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")]
|
||||
[CommandOption("aggregate", Description = "Aggregate function: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")]
|
||||
public string? Aggregate { get; init; }
|
||||
|
||||
/// <summary>
|
||||
@@ -56,6 +56,13 @@ public class HistoryReadCommand : CommandBase
|
||||
[CommandOption("interval", Description = "Processing interval in milliseconds for aggregates")]
|
||||
public double IntervalMs { get; init; } = 3600000;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether to read modified-value history (Part 11 IsReadModified) instead of raw samples:
|
||||
/// the prior value each HistoryUpdate replaced or deleted, with its update type, user and modification time.
|
||||
/// </summary>
|
||||
[CommandOption("modified", Description = "Read modified-value history (what HistoryUpdate changed: prior value, update type, user, time) instead of raw samples")]
|
||||
public bool Modified { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
||||
@@ -64,6 +71,8 @@ public class HistoryReadCommand : CommandBase
|
||||
throw new CommandException($"--max must be greater than 0 (was {MaxValues}).");
|
||||
if (!string.IsNullOrEmpty(Aggregate) && IntervalMs <= 0)
|
||||
throw new CommandException($"--interval must be greater than 0 (was {IntervalMs}).");
|
||||
if (Modified && !string.IsNullOrEmpty(Aggregate))
|
||||
throw new CommandException("--modified and --aggregate cannot be combined.");
|
||||
|
||||
NodeId nodeId;
|
||||
try
|
||||
@@ -119,6 +128,30 @@ public class HistoryReadCommand : CommandBase
|
||||
var ct = console.RegisterCancellationHandler();
|
||||
(service, _) = await CreateServiceAndConnectAsync(ct);
|
||||
|
||||
if (Modified)
|
||||
{
|
||||
await console.Output.WriteLineAsync(
|
||||
$"Modified history for {NodeId} ({start:yyyy-MM-dd HH:mm} -> {end:yyyy-MM-dd HH:mm})");
|
||||
var entries = await service.HistoryReadModifiedAsync(nodeId, start, end, MaxValues, ct);
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync(
|
||||
$"{"Timestamp",-35} {"Prior value",-15} {"Status",-10} {"Update",-8} {"User",-20} {"Modified at"}");
|
||||
foreach (var e in entries)
|
||||
{
|
||||
var dv = e.Value;
|
||||
var status = StatusCode.IsGood(dv.StatusCode) ? "Good"
|
||||
: StatusCode.IsBad(dv.StatusCode) ? "Bad"
|
||||
: "Uncertain";
|
||||
await console.Output.WriteLineAsync(
|
||||
$"{dv.SourceTimestamp.ToString("O"),-35} {dv.Value?.ToString() ?? "(none)",-15} {status,-10} " +
|
||||
$"{e.UpdateType,-8} {e.UserName ?? "",-20} {e.ModificationTime:O}");
|
||||
}
|
||||
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync($"{entries.Count} modifications returned.");
|
||||
return;
|
||||
}
|
||||
|
||||
IReadOnlyList<DataValue> values;
|
||||
|
||||
if (string.IsNullOrEmpty(Aggregate))
|
||||
@@ -150,6 +183,12 @@ public class HistoryReadCommand : CommandBase
|
||||
await console.Output.WriteLineAsync();
|
||||
await console.Output.WriteLineAsync($"{values.Count} values returned.");
|
||||
}
|
||||
catch (ServiceResultException ex) when (ex.InnerException is null)
|
||||
{
|
||||
// e.g. HistoryReadStatus.ThrowIfRefused: a refusal is an operator-facing outcome, not a crash.
|
||||
// A wrapped failure (certificate validation, identity rejection on connect) keeps its full detail.
|
||||
throw new CommandException(ex.Message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (service != null)
|
||||
@@ -171,8 +210,13 @@ public class HistoryReadCommand : CommandBase
|
||||
"start" or "first" => AggregateType.Start,
|
||||
"end" or "last" => AggregateType.End,
|
||||
"standarddeviation" or "stddev" or "stdev" => AggregateType.StandardDeviation,
|
||||
"timeaverage" or "timeavg" => AggregateType.TimeAverage,
|
||||
"timeaverage2" or "timeavg2" => AggregateType.TimeAverage2,
|
||||
"total" => AggregateType.Total,
|
||||
"total2" => AggregateType.Total2,
|
||||
"interpolative" or "interp" => AggregateType.Interpolative,
|
||||
_ => throw new ArgumentException(
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation")
|
||||
$"Unknown aggregate: '{name}'. Supported: Average, Minimum, Maximum, Count, Start, End, StandardDeviation, TimeAverage, TimeAverage2, Total, Total2, Interpolative")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
using System.Globalization;
|
||||
using CliFx.Attributes;
|
||||
using CliFx.Exceptions;
|
||||
using CliFx.Infrastructure;
|
||||
using Opc.Ua;
|
||||
using ZB.MOM.WW.OtOpcUa.Client.CLI.Helpers;
|
||||
using ZB.MOM.WW.OtOpcUa.Client.Shared;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Client.CLI.Commands;
|
||||
|
||||
[Command("historyupdate", Description = "Insert, replace, update or delete historical data on a node")]
|
||||
public class HistoryUpdateCommand : CommandBase
|
||||
{
|
||||
private static readonly string[] Modes = ["insert", "replace", "update", "delete-raw", "delete-at-time"];
|
||||
|
||||
/// <summary>
|
||||
/// Creates the history-update command used to modify stored history for a node.
|
||||
/// </summary>
|
||||
/// <param name="factory">The factory that creates the shared client service for the command run.</param>
|
||||
public HistoryUpdateCommand(IOpcUaClientServiceFactory factory) : base(factory)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Gets the historized node to modify.</summary>
|
||||
[CommandOption("node", 'n', Description = "Node ID (e.g. ns=2;s=MyNode)", IsRequired = true)]
|
||||
public string NodeId { get; init; } = default!;
|
||||
|
||||
/// <summary>Gets the update mode.</summary>
|
||||
[CommandOption("mode", Description = "insert | replace | update | delete-raw | delete-at-time", IsRequired = true)]
|
||||
public string Mode { get; init; } = default!;
|
||||
|
||||
/// <summary>Gets the values to write, paired positionally with <see cref="Times"/>.</summary>
|
||||
[CommandOption("value", Description = "Value to write (repeatable; pairs positionally with --time) for insert/replace/update")]
|
||||
public IReadOnlyList<string>? Values { get; init; }
|
||||
|
||||
/// <summary>Gets the source timestamps (ISO 8601 UTC), repeatable.</summary>
|
||||
[CommandOption("time", Description = "ISO 8601 UTC timestamp (repeatable) for insert/replace/update/delete-at-time")]
|
||||
public IReadOnlyList<string>? Times { get; init; }
|
||||
|
||||
/// <summary>Gets the value type used to parse <see cref="Values"/>.</summary>
|
||||
[CommandOption("type", Description = "Value type: double (default), int64, bool, string, datetime")]
|
||||
public string Type { get; init; } = "double";
|
||||
|
||||
/// <summary>Gets the start of the range for delete-raw.</summary>
|
||||
[CommandOption("start", Description = "Range start (ISO 8601 UTC) for delete-raw")]
|
||||
public string? StartTime { get; init; }
|
||||
|
||||
/// <summary>Gets the end of the range for delete-raw.</summary>
|
||||
[CommandOption("end", Description = "Range end (ISO 8601 UTC) for delete-raw")]
|
||||
public string? EndTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Validates the options and builds the HistoryUpdate details for the selected mode.
|
||||
/// </summary>
|
||||
/// <returns>The details object to send.</returns>
|
||||
/// <exception cref="CommandException">Thrown when options are missing, malformed or inconsistent.</exception>
|
||||
public HistoryUpdateDetails BuildDetails()
|
||||
{
|
||||
NodeId nodeId;
|
||||
try
|
||||
{
|
||||
nodeId = NodeIdParser.ParseRequired(NodeId);
|
||||
}
|
||||
catch (Exception ex) when (ex is FormatException or ArgumentException)
|
||||
{
|
||||
throw new CommandException($"Invalid --node value: {ex.Message}");
|
||||
}
|
||||
|
||||
var mode = (Mode ?? string.Empty).Trim().ToLowerInvariant();
|
||||
if (Array.IndexOf(Modes, mode) < 0)
|
||||
throw new CommandException(
|
||||
$"Invalid --mode value '{Mode}'. Supported: {string.Join(", ", Modes)}.");
|
||||
|
||||
var values = Values ?? [];
|
||||
var times = Times ?? [];
|
||||
var hasRange = !string.IsNullOrEmpty(StartTime) || !string.IsNullOrEmpty(EndTime);
|
||||
|
||||
switch (mode)
|
||||
{
|
||||
case "insert":
|
||||
case "replace":
|
||||
case "update":
|
||||
{
|
||||
if (hasRange)
|
||||
throw new CommandException($"--start/--end apply only to --mode delete-raw (mode was {mode}).");
|
||||
if (values.Count == 0)
|
||||
throw new CommandException($"--mode {mode} requires at least one --value/--time pair.");
|
||||
if (values.Count != times.Count)
|
||||
throw new CommandException(
|
||||
$"--value and --time must be supplied in pairs ({values.Count} value(s), {times.Count} time(s)).");
|
||||
|
||||
var updates = new DataValueCollection();
|
||||
for (var i = 0; i < values.Count; i++)
|
||||
{
|
||||
var ts = ParseTime(times[i], "--time");
|
||||
updates.Add(new DataValue(new Variant(ParseValue(values[i], Type)), StatusCodes.Good, ts, ts));
|
||||
}
|
||||
|
||||
return new UpdateDataDetails
|
||||
{
|
||||
NodeId = nodeId,
|
||||
PerformInsertReplace = mode switch
|
||||
{
|
||||
"insert" => PerformUpdateType.Insert,
|
||||
"replace" => PerformUpdateType.Replace,
|
||||
_ => PerformUpdateType.Update
|
||||
},
|
||||
UpdateValues = updates
|
||||
};
|
||||
}
|
||||
case "delete-raw":
|
||||
{
|
||||
if (values.Count > 0 || times.Count > 0)
|
||||
throw new CommandException("--mode delete-raw takes --start/--end, not --value/--time.");
|
||||
if (string.IsNullOrEmpty(StartTime) || string.IsNullOrEmpty(EndTime))
|
||||
throw new CommandException("--mode delete-raw requires both --start and --end.");
|
||||
var start = ParseTime(StartTime, "--start");
|
||||
var end = ParseTime(EndTime, "--end");
|
||||
if (end <= start)
|
||||
throw new CommandException("--end must be later than --start.");
|
||||
return new DeleteRawModifiedDetails
|
||||
{
|
||||
NodeId = nodeId,
|
||||
IsDeleteModified = false,
|
||||
StartTime = start,
|
||||
EndTime = end
|
||||
};
|
||||
}
|
||||
default: // delete-at-time
|
||||
{
|
||||
if (hasRange)
|
||||
throw new CommandException("--start/--end apply only to --mode delete-raw.");
|
||||
if (values.Count > 0)
|
||||
throw new CommandException("--mode delete-at-time takes --time only; --value is not allowed.");
|
||||
if (times.Count == 0)
|
||||
throw new CommandException("--mode delete-at-time requires at least one --time.");
|
||||
var req = new DateTimeCollection();
|
||||
foreach (var t in times) req.Add(ParseTime(t, "--time"));
|
||||
return new DeleteAtTimeDetails { NodeId = nodeId, ReqTimes = req };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async ValueTask ExecuteAsync(IConsole console)
|
||||
{
|
||||
ConfigureLogging();
|
||||
var details = BuildDetails();
|
||||
var times = DescribeTimes(details);
|
||||
|
||||
IOpcUaClientService? service = null;
|
||||
try
|
||||
{
|
||||
var ct = console.RegisterCancellationHandler();
|
||||
(service, _) = await CreateServiceAndConnectAsync(ct);
|
||||
|
||||
var outcome = await service.HistoryUpdateAsync(details, ct);
|
||||
|
||||
await console.Output.WriteLineAsync($"HistoryUpdate {Mode.Trim().ToLowerInvariant()} on {NodeId}");
|
||||
await console.Output.WriteLineAsync($"{"Index",-6} {"Time",-35} {"Status"}");
|
||||
|
||||
// Exit non-zero unless EVERY expected result came back Good: a Bad or Uncertain item, a Bad result or
|
||||
// service status, or a Good service result that is missing results (the server answered fewer items
|
||||
// than were sent) all fail the command.
|
||||
var failed = false;
|
||||
var expected = ExpectedOperationCount(details);
|
||||
var ops = outcome.OperationResults.Count > 0 ? outcome.OperationResults[0] : [];
|
||||
if (expected > 0)
|
||||
{
|
||||
for (var i = 0; i < Math.Max(expected, ops.Count); i++)
|
||||
{
|
||||
var time = i < times.Count ? times[i] : "-";
|
||||
string status;
|
||||
if (i < ops.Count)
|
||||
{
|
||||
failed |= !StatusCode.IsGood(ops[i]);
|
||||
status = StatusName(ops[i]);
|
||||
}
|
||||
else
|
||||
{
|
||||
failed = true;
|
||||
status = "(no result returned)";
|
||||
}
|
||||
await console.Output.WriteLineAsync($"{i,-6} {time,-35} {status}");
|
||||
}
|
||||
}
|
||||
else if (outcome.ResultStatus.Count > 0)
|
||||
{
|
||||
// delete-raw reports a single result-level status for the whole range.
|
||||
await console.Output.WriteLineAsync(
|
||||
$"{0,-6} {(times.Count > 0 ? times[0] : "-"),-35} {StatusName(outcome.ResultStatus[0])}");
|
||||
}
|
||||
|
||||
if (outcome.ResultStatus.Count == 0)
|
||||
{
|
||||
failed = true;
|
||||
await console.Output.WriteLineAsync("No history-update result was returned for the node.");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Print the node-level result too: a node the server refuses outright (e.g. BadUserAccessDenied)
|
||||
// carries its reason HERE with no per-item results, so the table alone shows only "(no result
|
||||
// returned)" and hides why.
|
||||
await console.Output.WriteLineAsync($"Node result: {StatusName(outcome.ResultStatus[0])}");
|
||||
failed |= !StatusCode.IsGood(outcome.ResultStatus[0]);
|
||||
}
|
||||
|
||||
await console.Output.WriteLineAsync($"Service result: {StatusName(outcome.ServiceResult)}");
|
||||
failed |= !StatusCode.IsGood(outcome.ServiceResult);
|
||||
|
||||
if (failed)
|
||||
throw new CommandException(
|
||||
"HistoryUpdate did not report Good for every item (a Bad or Uncertain result, or a missing result).");
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (service != null)
|
||||
{
|
||||
await service.DisconnectAsync();
|
||||
service.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>How many per-item operation results the server owes for <paramref name="details"/> (0 for delete-raw).</summary>
|
||||
private static int ExpectedOperationCount(HistoryUpdateDetails details) => details switch
|
||||
{
|
||||
UpdateDataDetails u => u.UpdateValues.Count,
|
||||
DeleteAtTimeDetails d => d.ReqTimes.Count,
|
||||
_ => 0
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The SDK's symbolic name for <paramref name="code"/>, or its hex value when the SDK has none — a status is
|
||||
/// never printed blank.
|
||||
/// </summary>
|
||||
/// <param name="code">The status code to describe.</param>
|
||||
/// <returns>The status name, or <c>0xXXXXXXXX</c>.</returns>
|
||||
private static string StatusName(StatusCode code)
|
||||
{
|
||||
var name = StatusCodes.GetBrowseName(code.Code);
|
||||
if (string.IsNullOrEmpty(name)) name = StatusCodes.GetBrowseName(code.Code & 0xFFFF0000u);
|
||||
return string.IsNullOrEmpty(name) ? $"0x{code.Code:X8}" : name;
|
||||
}
|
||||
|
||||
private static List<string> DescribeTimes(HistoryUpdateDetails details)
|
||||
{
|
||||
return details switch
|
||||
{
|
||||
UpdateDataDetails u => u.UpdateValues.Select(v => v.SourceTimestamp.ToString("O")).ToList(),
|
||||
DeleteAtTimeDetails d => d.ReqTimes.Select(t => t.ToString("O")).ToList(),
|
||||
DeleteRawModifiedDetails r => [$"{r.StartTime:O}..{r.EndTime:O}"],
|
||||
_ => []
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTime ParseTime(string text, string option)
|
||||
{
|
||||
if (!DateTime.TryParse(text, CultureInfo.InvariantCulture,
|
||||
DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
throw new CommandException(
|
||||
$"Invalid {option} value '{text}'. Expected ISO 8601 UTC format, e.g. 2026-01-15T08:00:00Z.");
|
||||
return parsed;
|
||||
}
|
||||
|
||||
private static object ParseValue(string text, string type)
|
||||
{
|
||||
var inv = CultureInfo.InvariantCulture;
|
||||
switch ((type ?? "double").Trim().ToLowerInvariant())
|
||||
{
|
||||
case "double":
|
||||
if (double.TryParse(text, NumberStyles.Float, inv, out var d)) return d;
|
||||
break;
|
||||
case "int64":
|
||||
if (long.TryParse(text, NumberStyles.Integer, inv, out var l)) return l;
|
||||
break;
|
||||
case "bool":
|
||||
if (bool.TryParse(text, out var b)) return b;
|
||||
if (text == "1") return true;
|
||||
if (text == "0") return false;
|
||||
break;
|
||||
case "string":
|
||||
return text;
|
||||
case "datetime":
|
||||
return ParseTime(text, "--value");
|
||||
default:
|
||||
throw new CommandException(
|
||||
$"Invalid --type value '{type}'. Supported: double, int64, bool, string, datetime.");
|
||||
}
|
||||
|
||||
throw new CommandException($"Invalid --value '{text}' for --type {type?.Trim().ToLowerInvariant()}.");
|
||||
}
|
||||
}
|
||||
@@ -85,17 +85,22 @@ internal static class EndpointSelector
|
||||
$"No endpoint found with security mode '{requestedMode}'. Available endpoints: {available}");
|
||||
}
|
||||
|
||||
// Rewrite endpoint URL hostname to match user-supplied hostname. Necessary
|
||||
// when the OPC UA server returns a discovery URL using a different hostname
|
||||
// (e.g. internal DNS name) than the one the operator routed to.
|
||||
// Rewrite the endpoint URL host AND port to the user-supplied ones (as the SDK's
|
||||
// CoreClientUtils.SelectEndpoint does). A server behind a port mapping or internal DNS
|
||||
// advertises its own host:port (docker-dev containers all say :4840); following that would
|
||||
// open the session on a different server and fail the secure-channel certificate check.
|
||||
// Only when the scheme matches: a different scheme (https vs opc.tcp) is a different transport.
|
||||
var serverUri = new Uri(best.EndpointUrl);
|
||||
var requestedUri = new Uri(endpointUrl);
|
||||
if (serverUri.Host != requestedUri.Host)
|
||||
// A URL typed without a port (opc.tcp has no registered default, so Port is -1) keeps the advertised port.
|
||||
var port = requestedUri.Port > 0 ? requestedUri.Port : serverUri.Port;
|
||||
if (string.Equals(serverUri.Scheme, requestedUri.Scheme, StringComparison.OrdinalIgnoreCase)
|
||||
&& (serverUri.Host != requestedUri.Host || serverUri.Port != port))
|
||||
{
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host };
|
||||
var builder = new UriBuilder(best.EndpointUrl) { Host = requestedUri.Host, Port = port };
|
||||
best.EndpointUrl = builder.ToString();
|
||||
Logger.Debug("Rewrote endpoint host from {ServerHost} to {RequestedHost}", serverUri.Host,
|
||||
requestedUri.Host);
|
||||
Logger.Debug("Rewrote endpoint from {ServerHost}:{ServerPort} to {RequestedHost}:{RequestedPort}",
|
||||
serverUri.Host, serverUri.Port, requestedUri.Host, port);
|
||||
}
|
||||
|
||||
return best;
|
||||
|
||||
@@ -131,6 +131,26 @@ internal sealed class DefaultSessionAdapter : ISessionAdapter
|
||||
return references != null && references.Count > 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<Models.HistoryUpdateOutcome> HistoryUpdateAsync(HistoryUpdateDetails details, CancellationToken ct)
|
||||
{
|
||||
var response = await _session.HistoryUpdateAsync(
|
||||
null,
|
||||
new ExtensionObjectCollection { new ExtensionObject(details) },
|
||||
ct).ConfigureAwait(false);
|
||||
|
||||
var statuses = new List<StatusCode>();
|
||||
var ops = new List<IReadOnlyList<StatusCode>>();
|
||||
if (response.Results != null)
|
||||
foreach (var r in response.Results)
|
||||
{
|
||||
statuses.Add(r.StatusCode);
|
||||
ops.Add(r.OperationResults?.ToArray() ?? Array.Empty<StatusCode>());
|
||||
}
|
||||
|
||||
return new Models.HistoryUpdateOutcome(response.ResponseHeader.ServiceResult, statuses, ops);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<DataValue>> HistoryReadRawAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues, CancellationToken ct)
|
||||
@@ -186,6 +206,74 @@ internal sealed class DefaultSessionAdapter : ISessionAdapter
|
||||
return allValues;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<Models.HistoryModifiedEntry>> HistoryReadModifiedAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues, CancellationToken ct)
|
||||
{
|
||||
var details = new ReadRawModifiedDetails
|
||||
{
|
||||
StartTime = startTime,
|
||||
EndTime = endTime,
|
||||
NumValuesPerNode = (uint)maxValues,
|
||||
IsReadModified = true,
|
||||
ReturnBounds = false
|
||||
};
|
||||
|
||||
var nodesToRead = new HistoryReadValueIdCollection
|
||||
{
|
||||
new HistoryReadValueId { NodeId = nodeId }
|
||||
};
|
||||
|
||||
var all = new List<Models.HistoryModifiedEntry>();
|
||||
byte[]? continuationPoint = null;
|
||||
|
||||
do
|
||||
{
|
||||
if (continuationPoint != null)
|
||||
nodesToRead[0].ContinuationPoint = continuationPoint;
|
||||
|
||||
var response = await _session.HistoryReadAsync(
|
||||
null,
|
||||
new ExtensionObject(details),
|
||||
TimestampsToReturn.Both,
|
||||
false,
|
||||
nodesToRead,
|
||||
ct).ConfigureAwait(false);
|
||||
|
||||
var results = response.Results;
|
||||
if (results == null || results.Count == 0)
|
||||
break;
|
||||
|
||||
var result = results[0];
|
||||
HistoryReadStatus.ThrowIfRefused(result.StatusCode, nodeId);
|
||||
|
||||
if (result.HistoryData is ExtensionObject ext && ext.Body is HistoryModifiedData modified)
|
||||
{
|
||||
for (var i = 0; i < modified.DataValues.Count; i++)
|
||||
{
|
||||
var info = i < modified.ModificationInfos.Count ? modified.ModificationInfos[i] : null;
|
||||
all.Add(new Models.HistoryModifiedEntry(
|
||||
modified.DataValues[i],
|
||||
info?.ModificationTime ?? DateTime.MinValue,
|
||||
info?.UpdateType ?? default,
|
||||
info?.UserName));
|
||||
}
|
||||
}
|
||||
|
||||
continuationPoint = result.ContinuationPoint;
|
||||
} while (continuationPoint != null && continuationPoint.Length > 0 && all.Count < maxValues);
|
||||
|
||||
// A read stopped by the client's own cap leaves a server cursor behind; release it.
|
||||
if (continuationPoint is { Length: > 0 })
|
||||
{
|
||||
nodesToRead[0].ContinuationPoint = continuationPoint;
|
||||
await _session.HistoryReadAsync(null, new ExtensionObject(details), TimestampsToReturn.Both, true,
|
||||
nodesToRead, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return all;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<DataValue>> HistoryReadAggregateAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, NodeId aggregateId, double intervalMs,
|
||||
|
||||
@@ -110,6 +110,29 @@ internal interface ISessionAdapter : IDisposable
|
||||
Task<IReadOnlyList<DataValue>> HistoryReadRawAsync(NodeId nodeId, DateTime startTime, DateTime endTime,
|
||||
int maxValues, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reads modified-value history (Part 11, <c>IsReadModified = true</c>): the prior values and modification info
|
||||
/// the server recorded for every HistoryUpdate in the window, following continuation points.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The historized node.</param>
|
||||
/// <param name="startTime">Window start.</param>
|
||||
/// <param name="endTime">Window end.</param>
|
||||
/// <param name="maxValues">The maximum number of entries to return.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The modification entries in server order.</returns>
|
||||
Task<IReadOnlyList<Models.HistoryModifiedEntry>> HistoryReadModifiedAsync(NodeId nodeId, DateTime startTime,
|
||||
DateTime endTime, int maxValues, CancellationToken ct = default) =>
|
||||
throw new NotSupportedException("This session adapter does not support modified-value history reads.");
|
||||
|
||||
/// <summary>
|
||||
/// Sends a HistoryUpdate request for the supplied details.
|
||||
/// </summary>
|
||||
/// <param name="details">The history update details.</param>
|
||||
/// <param name="ct">The cancellation token that aborts the call.</param>
|
||||
/// <returns>The overall service result and per-item status codes.</returns>
|
||||
Task<Models.HistoryUpdateOutcome> HistoryUpdateAsync(HistoryUpdateDetails details, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException("HistoryUpdate is not supported by this session adapter.");
|
||||
|
||||
/// <summary>
|
||||
/// Reads processed/aggregate historical data.
|
||||
/// </summary>
|
||||
|
||||
@@ -24,6 +24,11 @@ public static class AggregateTypeMapper
|
||||
AggregateType.Start => ObjectIds.AggregateFunction_Start,
|
||||
AggregateType.End => ObjectIds.AggregateFunction_End,
|
||||
AggregateType.StandardDeviation => ObjectIds.AggregateFunction_StandardDeviationPopulation,
|
||||
AggregateType.TimeAverage => ObjectIds.AggregateFunction_TimeAverage,
|
||||
AggregateType.TimeAverage2 => ObjectIds.AggregateFunction_TimeAverage2,
|
||||
AggregateType.Total => ObjectIds.AggregateFunction_Total,
|
||||
AggregateType.Total2 => ObjectIds.AggregateFunction_Total2,
|
||||
AggregateType.Interpolative => ObjectIds.AggregateFunction_Interpolative,
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(aggregate), aggregate, "Unknown AggregateType value.")
|
||||
};
|
||||
}
|
||||
|
||||
@@ -193,6 +193,29 @@ public interface IOpcUaClientService : IDisposable
|
||||
Task<IReadOnlyList<DataValue>> HistoryReadRawAsync(NodeId nodeId, DateTime startTime, DateTime endTime,
|
||||
int maxValues = 1000, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reads modified-value history (Part 11, <c>IsReadModified = true</c>): what each HistoryUpdate in the window
|
||||
/// replaced or deleted, with the server's modification time, update type and user name.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The historized node.</param>
|
||||
/// <param name="startTime">Window start.</param>
|
||||
/// <param name="endTime">Window end.</param>
|
||||
/// <param name="maxValues">The maximum number of entries to return.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The modification entries in server order.</returns>
|
||||
Task<IReadOnlyList<HistoryModifiedEntry>> HistoryReadModifiedAsync(NodeId nodeId, DateTime startTime,
|
||||
DateTime endTime, int maxValues = 1000, CancellationToken ct = default) =>
|
||||
throw new NotSupportedException("This client service does not support modified-value history reads.");
|
||||
|
||||
/// <summary>
|
||||
/// Sends a HistoryUpdate request (insert/replace/update or delete) for the supplied details.
|
||||
/// </summary>
|
||||
/// <param name="details">The history update details (UpdateDataDetails, DeleteRawModifiedDetails, DeleteAtTimeDetails...).</param>
|
||||
/// <param name="ct">The cancellation token that aborts the call.</param>
|
||||
/// <returns>The overall service result and per-item status codes.</returns>
|
||||
Task<HistoryUpdateOutcome> HistoryUpdateAsync(HistoryUpdateDetails details, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException("HistoryUpdate is not supported by this client service.");
|
||||
|
||||
/// <summary>
|
||||
/// Reads aggregate historical values for a historized node using an OPC UA aggregate function.
|
||||
/// </summary>
|
||||
|
||||
@@ -24,5 +24,20 @@ public enum AggregateType
|
||||
End,
|
||||
|
||||
/// <summary>Population standard deviation of values in the interval.</summary>
|
||||
StandardDeviation
|
||||
StandardDeviation,
|
||||
|
||||
/// <summary>Time-weighted average, interpolated bounds (Part 13 TimeAverage).</summary>
|
||||
TimeAverage,
|
||||
|
||||
/// <summary>Time-weighted average, stepped bounds (Part 13 TimeAverage2).</summary>
|
||||
TimeAverage2,
|
||||
|
||||
/// <summary>Time-weighted integral over the interval (Part 13 Total).</summary>
|
||||
Total,
|
||||
|
||||
/// <summary>Time-weighted integral, stepped bounds (Part 13 Total2).</summary>
|
||||
Total2,
|
||||
|
||||
/// <summary>Interpolated value at the interval start (Part 13 Interpolative).</summary>
|
||||
Interpolative
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Client.Shared.Models;
|
||||
|
||||
/// <summary>
|
||||
/// One entry of an OPC UA Part 11 modified-value history read (HistoryReadRaw with <c>IsReadModified = true</c>): the
|
||||
/// value as it was BEFORE the modification plus the server's <see cref="ModificationInfo"/>.
|
||||
/// </summary>
|
||||
/// <param name="Value">The prior value at its source timestamp (an insert had none: null value, Bad status).</param>
|
||||
/// <param name="ModificationTime">When the modification was applied.</param>
|
||||
/// <param name="UpdateType">The Part 11 update type (Insert / Replace / Update / Delete).</param>
|
||||
/// <param name="UserName">Who made the modification (the server's recorded principal).</param>
|
||||
public sealed record HistoryModifiedEntry(
|
||||
DataValue Value,
|
||||
DateTime ModificationTime,
|
||||
HistoryUpdateType UpdateType,
|
||||
string? UserName);
|
||||
@@ -0,0 +1,15 @@
|
||||
using Opc.Ua;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Client.Shared.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Result of a HistoryUpdate service call: the overall service result plus, per history-update result,
|
||||
/// the operation-level status codes (one per value / requested time) and the result's own status code.
|
||||
/// </summary>
|
||||
/// <param name="ServiceResult">The service-level result from the response header.</param>
|
||||
/// <param name="ResultStatus">The per-details-entry status codes.</param>
|
||||
/// <param name="OperationResults">Per-details-entry operation status codes (one per value or requested time; may be empty).</param>
|
||||
public sealed record HistoryUpdateOutcome(
|
||||
StatusCode ServiceResult,
|
||||
IReadOnlyList<StatusCode> ResultStatus,
|
||||
IReadOnlyList<IReadOnlyList<StatusCode>> OperationResults);
|
||||
@@ -559,6 +559,23 @@ public sealed class OpcUaClientService : IOpcUaClientService
|
||||
return await _session!.HistoryReadRawAsync(nodeId, startTime, endTime, maxValues, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<HistoryModifiedEntry>> HistoryReadModifiedAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, int maxValues = 1000, CancellationToken ct = default)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
ThrowIfNotConnected();
|
||||
return await _session!.HistoryReadModifiedAsync(nodeId, startTime, endTime, maxValues, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoryUpdateOutcome> HistoryUpdateAsync(HistoryUpdateDetails details, CancellationToken ct = default)
|
||||
{
|
||||
ThrowIfDisposed();
|
||||
ThrowIfNotConnected();
|
||||
return await _session!.HistoryUpdateAsync(details, ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<DataValue>> HistoryReadAggregateAsync(
|
||||
NodeId nodeId, DateTime startTime, DateTime endTime, AggregateType aggregate,
|
||||
|
||||
@@ -53,7 +53,12 @@ public partial class HistoryViewModel : ObservableObject
|
||||
AggregateType.Count,
|
||||
AggregateType.Start,
|
||||
AggregateType.End,
|
||||
AggregateType.StandardDeviation
|
||||
AggregateType.StandardDeviation,
|
||||
AggregateType.TimeAverage,
|
||||
AggregateType.TimeAverage2,
|
||||
AggregateType.Total,
|
||||
AggregateType.Total2,
|
||||
AggregateType.Interpolative
|
||||
];
|
||||
|
||||
/// <summary>Gets a value indicating whether an aggregate read is selected.</summary>
|
||||
|
||||
@@ -80,11 +80,11 @@ public static class OtOpcUaTelemetry
|
||||
/// Session/publish-level events of an OPC UA Client driver instance (#276): the operator's view of a
|
||||
/// flapping upstream. Tagged <see cref="PollOverrunDriverInstanceTag"/> (<c>driver_instance</c>) +
|
||||
/// <c>event</c> = <c>publish_response|notification|data_change|bad_status_value|publish_error|
|
||||
/// keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted</c>.
|
||||
/// keepalive_miss|keepalive_recovered|stale_notification_dropped</c>.
|
||||
/// </summary>
|
||||
public static readonly Counter<long> OpcUaClientEvents =
|
||||
Meter.CreateCounter<long>("otopcua.driver.opcuaclient.events", unit: "{event}",
|
||||
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|reconnect_attempt|reconnect_success|reconnect_exhausted).");
|
||||
description: "OPC UA Client driver session/publish events (event=publish_response|notification|data_change|bad_status_value|publish_error|keepalive_miss|keepalive_recovered|stale_notification_dropped).");
|
||||
|
||||
// ---------------- VirtualTag / ScriptedAlarm engines ----------------
|
||||
|
||||
|
||||
@@ -30,8 +30,8 @@ public enum NodePermissions : int
|
||||
MethodCall = 1 << 11,
|
||||
|
||||
// OPC UA HistoryUpdate (annotation / insert / delete) — separate from HistoryRead so a
|
||||
// read-only grant cannot authorize historian writes. Not included in any composite bundle
|
||||
// until the HistoryUpdate service surface is implemented.
|
||||
// read-only grant cannot authorize historian writes. Deliberately not included in any
|
||||
// composite bundle — it must be granted explicitly.
|
||||
HistoryUpdate = 1 << 12,
|
||||
|
||||
// Bundles (one-click grants in Admin UI)
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>The value kinds a historian backend can store. Persisted as a byte (SQL ValueKind, outbox v2) — never renumber.</summary>
|
||||
public enum HistorianValueKind : byte { Empty = 0, Double = 1, Int64 = 2, Boolean = 3, String = 4, DateTime = 5 }
|
||||
|
||||
/// <summary>A typed historized value. Number carries Double (and the double view of Int64/Boolean);
|
||||
/// Integer carries Int64, Boolean (1/0) and DateTime (UTC ticks); Text carries String.</summary>
|
||||
/// <param name="Kind">Which payload field(s) are meaningful.</param>
|
||||
/// <param name="Number">Double payload (also the double view of Int64/Boolean).</param>
|
||||
/// <param name="Integer">Int64 payload, Boolean (1/0) or DateTime UTC ticks.</param>
|
||||
/// <param name="Text">String payload.</param>
|
||||
public readonly record struct HistorianValue(HistorianValueKind Kind, double Number, long Integer, string? Text)
|
||||
{
|
||||
/// <summary>No value (e.g. a Bad/Uncertain sample carrying null).</summary>
|
||||
public static HistorianValue Empty => default;
|
||||
|
||||
/// <summary>Creates a Double value.</summary>
|
||||
/// <param name="v">The value.</param>
|
||||
/// <returns>The typed value.</returns>
|
||||
public static HistorianValue FromDouble(double v) => new(HistorianValueKind.Double, v, 0, null);
|
||||
|
||||
/// <summary>Creates an Int64 value.</summary>
|
||||
/// <param name="v">The value.</param>
|
||||
/// <returns>The typed value.</returns>
|
||||
public static HistorianValue FromInt64(long v) => new(HistorianValueKind.Int64, v, v, null);
|
||||
|
||||
/// <summary>Creates a Boolean value (Integer and Number are 1/0).</summary>
|
||||
/// <param name="v">The value.</param>
|
||||
/// <returns>The typed value.</returns>
|
||||
public static HistorianValue FromBoolean(bool v) => new(HistorianValueKind.Boolean, v ? 1d : 0d, v ? 1 : 0, null);
|
||||
|
||||
/// <summary>Creates a String value.</summary>
|
||||
/// <param name="v">The value; must not be null.</param>
|
||||
/// <returns>The typed value.</returns>
|
||||
public static HistorianValue FromString(string v) =>
|
||||
new(HistorianValueKind.String, 0d, 0, v ?? throw new ArgumentNullException(nameof(v)));
|
||||
|
||||
/// <summary>Creates a DateTime value stored as UTC ticks (Local is converted, Unspecified is taken as UTC).</summary>
|
||||
/// <param name="v">The value.</param>
|
||||
/// <returns>The typed value.</returns>
|
||||
public static HistorianValue FromDateTime(DateTime v) =>
|
||||
new(HistorianValueKind.DateTime, 0d, (v.Kind == DateTimeKind.Local ? v.ToUniversalTime() : DateTime.SpecifyKind(v, DateTimeKind.Utc)).Ticks, null);
|
||||
|
||||
/// <summary>Coerces a CLR value into a typed historian value.</summary>
|
||||
/// <param name="value">The CLR value (null maps to <see cref="Empty"/>).</param>
|
||||
/// <param name="result">The typed value, or <see cref="Empty"/> when not historizable.</param>
|
||||
/// <returns><c>false</c> for arrays, references and any other non-historizable type.</returns>
|
||||
public static bool TryFromObject(object? value, out HistorianValue result)
|
||||
{
|
||||
switch (value)
|
||||
{
|
||||
case null: result = Empty; return true;
|
||||
case double d: result = FromDouble(d); return true;
|
||||
case float f: result = FromDouble(f); return true;
|
||||
case decimal m: result = FromDouble((double)m); return true;
|
||||
case sbyte sb: result = FromInt64(sb); return true;
|
||||
case byte b: result = FromInt64(b); return true;
|
||||
case short s: result = FromInt64(s); return true;
|
||||
case ushort us: result = FromInt64(us); return true;
|
||||
case int i: result = FromInt64(i); return true;
|
||||
case uint ui: result = FromInt64(ui); return true;
|
||||
case long l: result = FromInt64(l); return true;
|
||||
case ulong ul: result = ul <= long.MaxValue ? FromInt64((long)ul) : FromDouble(ul); return true;
|
||||
case bool bo: result = FromBoolean(bo); return true;
|
||||
case string str: result = FromString(str); return true;
|
||||
case DateTime dt: result = FromDateTime(dt); return true;
|
||||
default: result = Empty; return false; // arrays, references, structures => not historizable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The double view of the value.</summary>
|
||||
/// <param name="value">The double view; 0 when unavailable.</param>
|
||||
/// <returns><c>true</c> for Double, Int64 and Boolean kinds.</returns>
|
||||
public bool TryGetDouble(out double value)
|
||||
{
|
||||
switch (Kind)
|
||||
{
|
||||
case HistorianValueKind.Double:
|
||||
case HistorianValueKind.Int64:
|
||||
case HistorianValueKind.Boolean: value = Number; return true;
|
||||
default: value = 0d; return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Converts back to a CLR value (DateTime is returned as <see cref="DateTimeKind.Utc"/>).</summary>
|
||||
/// <returns>The boxed CLR value, or null for <see cref="HistorianValueKind.Empty"/>.</returns>
|
||||
public object? ToObject() => Kind switch
|
||||
{
|
||||
HistorianValueKind.Double => Number,
|
||||
HistorianValueKind.Int64 => Integer,
|
||||
HistorianValueKind.Boolean => Integer != 0,
|
||||
HistorianValueKind.String => Text,
|
||||
HistorianValueKind.DateTime => new DateTime(Integer, DateTimeKind.Utc),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -2,17 +2,16 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// One durable record buffered by the continuous-historization outbox before it is written to
|
||||
/// the historian. Carries the minimal payload the SQL analog live-value write path can ingest:
|
||||
/// a numeric value, a quality code, and a UTC timestamp keyed by tag.
|
||||
/// the historian: a typed value, an OPC UA status code, and a UTC timestamp keyed by tag.
|
||||
/// </summary>
|
||||
/// <param name="Id">Stable identifier used to ack (remove) the entry once written. Unique per append.</param>
|
||||
/// <param name="Tag">Fully-qualified historian tag name the value is recorded against.</param>
|
||||
/// <param name="NumericValue">The coerced numeric sample value (the SQL write path is numeric-only).</param>
|
||||
/// <param name="Quality">OPC-UA-derived quality code (e.g. 192 = Good) carried through to the historian.</param>
|
||||
/// <param name="Value">The typed sample value.</param>
|
||||
/// <param name="StatusCode">OPC UA status code of the sample (0 = Good).</param>
|
||||
/// <param name="TimestampUtc">UTC source timestamp of the sample.</param>
|
||||
public sealed record HistorizationOutboxEntry(
|
||||
Guid Id,
|
||||
string Tag,
|
||||
double NumericValue,
|
||||
ushort Quality,
|
||||
HistorianValue Value,
|
||||
uint StatusCode,
|
||||
DateTime TimestampUtc);
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
// OPC UA Part 11 modified-value history (Gitea #668): every HistoryUpdate a backend applies leaves an audit row
|
||||
// carrying the PRIOR value, who made the change and when; HistoryRead with IsReadModified=true serves them.
|
||||
|
||||
/// <summary>Who is performing a HistoryUpdate. Threaded into every <see cref="IHistorianDataUpdater"/> call.</summary>
|
||||
/// <param name="Principal">
|
||||
/// The session's authenticated principal (an API key's principal, or the reserved <c>anonymous</c>). Recorded as
|
||||
/// the modification's user name.
|
||||
/// </param>
|
||||
/// <param name="ApiKeyId">The API key id the session authenticated with, or <c>null</c> (anonymous / non-key).</param>
|
||||
public sealed record HistoryUpdateContext(string Principal, string? ApiKeyId)
|
||||
{
|
||||
/// <summary>The longest principal / key id a backend stores; longer values are truncated (never refused).</summary>
|
||||
public const int MaxIdentityLength = 200;
|
||||
|
||||
/// <summary>The principal clamped to <see cref="MaxIdentityLength"/> characters.</summary>
|
||||
public string StoredPrincipal => Clamp(Principal) ?? string.Empty;
|
||||
|
||||
/// <summary>The key id clamped to <see cref="MaxIdentityLength"/> characters.</summary>
|
||||
public string? StoredApiKeyId => Clamp(ApiKeyId);
|
||||
|
||||
private static string? Clamp(string? s) => s is { Length: > MaxIdentityLength } ? s[..MaxIdentityLength] : s;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The OPC UA Part 11 <c>HistoryUpdateType</c> of a recorded modification. The numeric values are the spec's and
|
||||
/// are persisted (SQL <c>tinyint</c>, QuestDB <c>BYTE</c>) — never renumber.
|
||||
/// </summary>
|
||||
public enum HistoryModificationType : byte
|
||||
{
|
||||
/// <summary>A sample was inserted where none existed (no prior value).</summary>
|
||||
Insert = 1,
|
||||
|
||||
/// <summary>An existing sample was replaced.</summary>
|
||||
Replace = 2,
|
||||
|
||||
/// <summary>An upsert (<c>PerformUpdateType.Update</c>): replaced a sample, or inserted one (then no prior value).</summary>
|
||||
Update = 3,
|
||||
|
||||
/// <summary>A sample was deleted (DeleteAtTime / DeleteRaw).</summary>
|
||||
Delete = 4,
|
||||
}
|
||||
|
||||
/// <summary>One recorded modification: the value as it was BEFORE the change, plus who changed it and when.</summary>
|
||||
/// <param name="Value">
|
||||
/// The prior value at its source timestamp. An insert had no prior value: <c>Value = null</c>,
|
||||
/// <c>StatusCode = BadNoData</c>.
|
||||
/// </param>
|
||||
/// <param name="ModificationTimeUtc">When the modification was applied.</param>
|
||||
/// <param name="UpdateType">The kind of modification.</param>
|
||||
/// <param name="UserName">The principal that made it.</param>
|
||||
/// <param name="ModifiedId">The backend's ordering key within one source timestamp (paging cursor).</param>
|
||||
/// <param name="SourceTimeUtc">
|
||||
/// The modified sample's source time as stored on the trail row — the paging cursor's time component. Explicit
|
||||
/// because an insert's <see cref="Value"/> has no prior value and so cannot be relied on to carry it.
|
||||
/// </param>
|
||||
public sealed record HistoryModifiedValue(
|
||||
DataValueSnapshot Value,
|
||||
DateTime ModificationTimeUtc,
|
||||
HistoryModificationType UpdateType,
|
||||
string UserName,
|
||||
long ModifiedId,
|
||||
DateTime SourceTimeUtc);
|
||||
|
||||
/// <summary>Resume position for a paged modified-value read: strictly after (or before, reversed) this key.</summary>
|
||||
/// <param name="SourceTimeUtc">The last returned modification's source time.</param>
|
||||
/// <param name="ModifiedId">The last returned modification's id.</param>
|
||||
public sealed record HistoryModifiedCursor(DateTime SourceTimeUtc, long ModifiedId);
|
||||
|
||||
/// <summary>Result of <see cref="IHistorianDataSource.ReadModifiedAsync"/>.</summary>
|
||||
/// <param name="Values">Modifications ordered by <c>(SourceTimeUtc, ModifiedId)</c> (descending for a reverse read).</param>
|
||||
/// <param name="HasMore"><c>true</c> when the cap truncated the result and more modifications remain in the window.</param>
|
||||
public sealed record HistoryModifiedReadResult(IReadOnlyList<HistoryModifiedValue> Values, bool HasMore);
|
||||
@@ -47,6 +47,24 @@ public enum HistoryAggregateType
|
||||
AnnotationCount,
|
||||
StandardDeviationPopulation,
|
||||
VariancePopulation,
|
||||
|
||||
/// <summary>Part 13 TimeAverage2: time-weighted average over the interval's non-Bad data only (#671).</summary>
|
||||
TimeAverage2,
|
||||
|
||||
/// <summary>Part 13 Total2: time-weighted integral (value·seconds) over the interval's non-Bad data only (#671).</summary>
|
||||
Total2,
|
||||
}
|
||||
|
||||
/// <summary>Shared <see cref="HistoryAggregateType"/> sets for <see cref="IHistorianDataSource.SupportedAggregates"/> (#682).</summary>
|
||||
public static class HistoryAggregateTypes
|
||||
{
|
||||
/// <summary>Every defined <see cref="HistoryAggregateType"/> — the interface default.</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> All =
|
||||
System.Collections.Frozen.FrozenSet.ToFrozenSet(Enum.GetValues<HistoryAggregateType>());
|
||||
|
||||
/// <summary>No aggregates — a source with no history to aggregate (the Null historian).</summary>
|
||||
public static readonly IReadOnlySet<HistoryAggregateType> None =
|
||||
System.Collections.Frozen.FrozenSet<HistoryAggregateType>.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -66,7 +84,31 @@ public sealed class HistoryAggregateNotSupportedException : NotSupportedExceptio
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync"/> — a historical
|
||||
/// Thrown by a history backend when a read would exceed a configured size limit (e.g. the raw-row cap of the SQL
|
||||
/// backends' time-weighted aggregates, #671). The Server HistoryRead path translates it to
|
||||
/// <c>BadTooManyOperations</c> for that node only; it is a client-request limit, not a backend fault.
|
||||
/// </summary>
|
||||
public sealed class HistoryReadLimitExceededException : InvalidOperationException
|
||||
{
|
||||
/// <summary>The configured limit that was exceeded.</summary>
|
||||
public long Limit { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #694: a raw read refused because rows BEFORE its window (inside the backend's look-back margin — a tie
|
||||
/// cluster just before a page start, the HistorianGateway's boundary row) crowded a ceiling-bounded row budget: how
|
||||
/// many such rows the read saw (a lower bound when the read was itself truncated). <c>0</c> = not a crowding refusal.
|
||||
/// The server re-reads such a page once, smaller by this many values (<c>RawPageLimit</c>).
|
||||
/// </summary>
|
||||
public int PreWindowRows { get; init; }
|
||||
|
||||
/// <summary>Creates the exception.</summary>
|
||||
/// <param name="limit">The configured limit.</param>
|
||||
/// <param name="message">Operator-facing description.</param>
|
||||
public HistoryReadLimitExceededException(long limit, string message) : base(message) => Limit = limit;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> — a historical
|
||||
/// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the
|
||||
/// Server needs to populate a <c>HistoryEventFieldList</c> for HistoryReadEvents responses.
|
||||
/// </summary>
|
||||
@@ -84,8 +126,8 @@ public sealed record HistoricalEvent(
|
||||
string? Message,
|
||||
ushort Severity);
|
||||
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync"/> call.</summary>
|
||||
/// <param name="Events">Events in chronological order by <c>EventTimeUtc</c>.</param>
|
||||
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> call.</summary>
|
||||
/// <param name="Events">Events ordered by <c>EventTimeUtc</c> in the read's direction — ascending, or descending for a reverse read (#675).</param>
|
||||
/// <param name="ContinuationPoint">Opaque token for the next call when more events are available; null when complete.</param>
|
||||
public sealed record HistoricalEventsResult(
|
||||
IReadOnlyList<HistoricalEvent> Events,
|
||||
|
||||
@@ -17,6 +17,12 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <summary>
|
||||
/// Read raw historical samples for a single tag over a time range.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Forward (<c>startUtc < endUtc</c>) is <c>[start, end)</c> ascending; <c>startUtc == endUtc</c> is a point read.
|
||||
/// <c>startUtc > endUtc</c> is the Part 11 reverse read <c>(end, start]</c> newest-first — but only a backend that
|
||||
/// declares <see cref="SupportsReverseRaw"/> is ever sent one (Gitea #686); the server serves a reverse read on any
|
||||
/// other backend from bounded forward reads.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="startUtc">The start of the time range in UTC.</param>
|
||||
/// <param name="endUtc">The end of the time range in UTC.</param>
|
||||
@@ -30,6 +36,28 @@ public interface IHistorianDataSource : IDisposable
|
||||
uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) raw read itself — <see cref="ReadRawAsync"/> with
|
||||
/// <c>startUtc > endUtc</c> (Gitea #686). The server checks it before every raw read. When <c>false</c> it never
|
||||
/// sends <c>start > end</c>: it serves the reverse read — an explicit one, a resumed reverse page, and the
|
||||
/// end-only capped read Part 11 reads backward from EndTime — from a series of bounded FORWARD reads walking back
|
||||
/// from the later bound (<c>ReverseRawReader</c> in the OPC UA server). Default <c>false</c>, the safe direction: a
|
||||
/// source that only reads forward is still answered correctly. A backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
bool SupportsReverseRaw => false;
|
||||
|
||||
/// <summary>
|
||||
/// The most values one capped <see cref="ReadRawAsync"/> can currently return without failing
|
||||
/// <see cref="HistoryReadLimitExceededException"/> on a BACKEND row ceiling (Gitea #694); <c>0</c> = no known limit.
|
||||
/// The server sizes its raw pages to it — a page is at most this many values and carries a continuation point when
|
||||
/// full (Part 11 allows fewer than <c>NumValuesPerNode</c> with a continuation point), instead of failing a page size
|
||||
/// the backend cannot read in one go. It may change at runtime (a strict HistorianGateway's ceiling is learned from
|
||||
/// its first refusal, #691.3), so it is read per read. A backend whose reads are never ceiling-bound returns <c>0</c>.
|
||||
/// The source still never truncates: a read asking for more than this is answered in full or refused. Default
|
||||
/// <c>0</c>; a backend declares it itself (a guard test pins that).
|
||||
/// </summary>
|
||||
int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Read processed (interval-bucketed) samples — average / min / max / count / etc.
|
||||
/// A bucket with no source data returns a sample whose
|
||||
@@ -65,6 +93,31 @@ public interface IHistorianDataSource : IDisposable
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> with the value type of
|
||||
/// the node being read, for a backend whose stored values do not carry their type (the Gateway: AVEVA stores every
|
||||
/// historized type as a numeric analog, so a Boolean comes back as a 0/1 Double). Such a backend uses the hint to apply
|
||||
/// the AtTime rules exactly as the typed backends do — a Boolean (or String / DateTime) node is stepped, an integer node
|
||||
/// is interpolated, rounded half away from zero and returned as the node's integer type, a floating-point node is
|
||||
/// interpolated. A backend that stores typed values ignores it: the default forwards to the hint-less overload, so
|
||||
/// SqlServer, QuestDB and the Null source need no change.
|
||||
/// </summary>
|
||||
/// <param name="fullReference">The full reference of the tag to read.</param>
|
||||
/// <param name="timestampsUtc">The list of timestamps to read values at.</param>
|
||||
/// <param name="valueType">
|
||||
/// The scalar value type of the node (<see cref="TypeCode.Boolean"/>, the integer codes, <see cref="TypeCode.Single"/>,
|
||||
/// <see cref="TypeCode.Double"/>, <see cref="TypeCode.String"/>, <see cref="TypeCode.DateTime"/>), or <c>null</c> when
|
||||
/// unknown, not scalar, or not one of those — the backend's own policy then applies.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">A cancellation token that can be used to cancel the operation.</param>
|
||||
/// <returns>One sample per requested timestamp, in the same order as <paramref name="timestampsUtc"/>.</returns>
|
||||
Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference,
|
||||
IReadOnlyList<DateTime> timestampsUtc,
|
||||
TypeCode? valueType,
|
||||
CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records — OPC UA HistoryReadEvents service.
|
||||
/// Distinct from any live event stream; sources here come from the historian's
|
||||
@@ -79,7 +132,7 @@ public interface IHistorianDataSource : IDisposable
|
||||
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
|
||||
/// equivalent "use default" sentinel.
|
||||
///
|
||||
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync"/>
|
||||
/// This surface declares <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
|
||||
/// as required members — a server-side historian owns the full read surface.
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
@@ -101,10 +154,123 @@ public interface IHistorianDataSource : IDisposable
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Whether this backend can serve a reverse (newest-first) events read — the reverse overload of
|
||||
/// <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, bool, CancellationToken)"/> (Gitea #675). The server
|
||||
/// checks it BEFORE calling the source: a reverse read (explicit <c>StartTime > EndTime</c>, or an end-only capped
|
||||
/// read, which Part 11 reads backward) on a backend that cannot is never sent the reverse overload — since Gitea
|
||||
/// #691.4 the server walks it back from bounded FORWARD reads instead (#675 refused it). Default <c>false</c>; an
|
||||
/// implementation (or a wrapper) declares it itself — a guard test pins that.
|
||||
/// </summary>
|
||||
bool SupportsReverseEvents => false;
|
||||
|
||||
/// <summary>
|
||||
/// Read historical alarm / event records in a chosen direction — the overload the server's HistoryReadEvents
|
||||
/// paging calls (Gitea #675).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The bounds may be given in either order; let <c>lo</c>/<c>hi</c> be the earlier/later of them. Forward
|
||||
/// (<paramref name="reverse"/> <c>false</c>) is exactly <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>:
|
||||
/// <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>. Reverse is the Part 11 reverse window
|
||||
/// <c>(lo, hi]</c> in DESCENDING <c>(EventTimeUtc, EventId)</c> order, so a cap keeps the LATEST
|
||||
/// <paramref name="maxEvents"/> events — the mirror of forward, with the same tie-breaker reversed, so a
|
||||
/// resumed reverse page that re-reads its boundary timestamp sees the boundary ties in the same order.
|
||||
/// <paramref name="maxEvents"/> and the continuation-point contract are as for the forward read.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The default forwards a forward read to the five-argument overload and throws
|
||||
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (the
|
||||
/// Gateway). The server never relies on that throw: it consults <see cref="SupportsReverseEvents"/> first and walks
|
||||
/// such a read back from forward reads (#691.4), never returning the earliest events in ascending order as if
|
||||
/// they were the answer. A backend that serves reverse natively must override BOTH members.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
|
||||
/// <param name="startUtc">One bound of the window (UTC).</param>
|
||||
/// <param name="endUtc">The other bound of the window (UTC).</param>
|
||||
/// <param name="maxEvents">The cap; non-positive uses the backend default (see the five-argument overload).</param>
|
||||
/// <param name="reverse"><c>true</c> to read newest-first over <c>(lo, hi]</c>.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>The matching events in the read's direction, with a continuation point if truncated.</returns>
|
||||
/// <exception cref="NotSupportedException">A reverse read on a backend that cannot read newest-first.</exception>
|
||||
Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException("This historian backend cannot read events newest-first (reverse).")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Point-in-time health snapshot for diagnostics and dashboards. Pure
|
||||
/// observation; never blocks on backend I/O.
|
||||
/// </summary>
|
||||
/// <returns>The current health snapshot for the historian backend.</returns>
|
||||
HistorianHealthSnapshot GetHealthSnapshot();
|
||||
|
||||
/// <summary>
|
||||
/// The backend's storage timestamp resolution, in <see cref="DateTime"/> ticks (100 ns): the smallest step
|
||||
/// between two distinct stored timestamps. <c>1</c> (the default) means full tick precision; QuestDB stores
|
||||
/// microseconds and reports <c>10</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The server's HistoryRead-Raw paging steps OFF a drained timestamp by this amount (forward
|
||||
/// <c>T + resolution</c>, reverse <c>T − resolution</c>). Stepping by less than the resolution is not
|
||||
/// progress on a backend that truncates its inputs: <c>T + 1 tick</c> is truncated back to <c>T</c>, the
|
||||
/// same sample comes back, and a one-value-per-page forward read loops forever. Stepping by the resolution is
|
||||
/// still lossless, because no stored timestamp can lie strictly between <c>T</c> and <c>T ± resolution</c>.
|
||||
/// Same-timestamp ties are unaffected (they are paged within the timestamp, never stepped over).
|
||||
/// </remarks>
|
||||
long TimestampResolutionTicks => 1;
|
||||
|
||||
/// <summary>
|
||||
/// The Part 13 aggregates this backend serves for a processed HistoryRead (Gitea #682) — the ONE authority for
|
||||
/// both what the server advertises under <c>HistoryServerCapabilities/AggregateFunctions</c> (i=11201) and what
|
||||
/// its HistoryRead-Processed answers <c>BadAggregateNotSupported</c> up front without calling the backend.
|
||||
/// (<c>ServerCapabilities/AggregateFunctions</c>, i=2997, is not affected: it lists every aggregate the server
|
||||
/// computes, including for live <c>AggregateFilter</c> subscriptions.)
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default is <see cref="HistoryAggregateTypes.All"/> — "every aggregate the server can name; a member I cannot
|
||||
/// serve throws <see cref="HistoryAggregateNotSupportedException"/> at read time", which is exactly the pre-#682
|
||||
/// behaviour, so a test fake or an out-of-tree source keeps working unchanged. It is a concrete set rather than a
|
||||
/// <c>null</c> "all" sentinel so every consumer handles one shape. Every PRODUCTION backend declares its set itself
|
||||
/// (a reflection guard pins that): a wrapper that forgot to forward this would otherwise advertise every aggregate.
|
||||
/// The set is read once when the source is wired (the backend is fixed at boot by <c>ServerHistorian:Backend</c>)
|
||||
/// and again on every processed read, so it must be cheap and stable — return a static frozen set. A source that
|
||||
/// returns <c>null</c> anyway is treated as the default (all) by both consumers.
|
||||
/// </remarks>
|
||||
IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.All;
|
||||
|
||||
/// <summary>
|
||||
/// Read modified-value history — OPC UA HistoryReadRaw with <c>IsReadModified = true</c> (Part 11, Gitea #668):
|
||||
/// every modification a HistoryUpdate applied to the tag inside the window, each carrying the PRIOR value.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Window: forward (<c>start < end</c>) <c>[start, end)</c> ascending by <c>(SourceTimeUtc, ModifiedId)</c>;
|
||||
/// reverse (<c>end < start</c>) <c>(end, start]</c> descending; <c>start == end</c> the modifications at
|
||||
/// exactly that time. <paramref name="resumeAfter"/> continues strictly past that key in the read's direction.
|
||||
/// The default throws <see cref="NotSupportedException"/> — a backend that keeps no modification trail
|
||||
/// (Gateway, Null) — which the server reports as <c>BadHistoryOperationUnsupported</c>.
|
||||
/// </remarks>
|
||||
/// <param name="fullReference">The historian tagname.</param>
|
||||
/// <param name="startUtc">Window start (UTC).</param>
|
||||
/// <param name="endUtc">Window end (UTC).</param>
|
||||
/// <param name="maxValues">Cap on returned modifications; <c>0</c> is unbounded.</param>
|
||||
/// <param name="resumeAfter">Resume cursor from a previous page, or <c>null</c> for the first page.</param>
|
||||
/// <param name="cancellationToken">Cancellation.</param>
|
||||
/// <returns>The modifications, with <see cref="HistoryModifiedReadResult.HasMore"/> set when truncated.</returns>
|
||||
/// <exception cref="NotSupportedException">The backend keeps no modified-value history.</exception>
|
||||
Task<HistoryModifiedReadResult> ReadModifiedAsync(
|
||||
string fullReference,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
uint maxValues,
|
||||
HistoryModifiedCursor? resumeAfter,
|
||||
CancellationToken cancellationToken) =>
|
||||
throw new NotSupportedException("This historian backend keeps no modified-value history.");
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>OPC UA Part 11 PerformUpdateType minus Remove.</summary>
|
||||
public enum HistoryUpdateMode
|
||||
{
|
||||
/// <summary>Insert new samples; an existing sample at the timestamp is refused.</summary>
|
||||
Insert,
|
||||
|
||||
/// <summary>Replace existing samples; a missing sample is refused.</summary>
|
||||
Replace,
|
||||
|
||||
/// <summary>Insert or replace (upsert).</summary>
|
||||
Update,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Optional historian capability behind OPC UA HistoryUpdate. A backend that cannot modify history
|
||||
/// supplies no updater (the server then reports HistoryUpdate as unsupported).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Modified-value history (Part 11, Gitea #668).</b> Every sample an implementation inserts, replaces or deletes
|
||||
/// MUST leave one modification record carrying the prior value (none for an insert), the
|
||||
/// <see cref="HistoryUpdateContext.Principal"/>, the modification time and the <see cref="HistoryModificationType"/>,
|
||||
/// readable through <see cref="IHistorianDataSource.ReadModifiedAsync"/>. A refused item (any non-Good code) leaves
|
||||
/// no record. Live recorder writes and retention purges never leave one.
|
||||
/// <para>
|
||||
/// <b>Cannot update at all</b> (Gitea #674): an implementation that is wired but cannot perform any update right
|
||||
/// now — the SqlServer backend on a schema without the modification trail — throws
|
||||
/// <see cref="NotSupportedException"/> from every method without touching storage. The server reports it as a
|
||||
/// node-level <c>BadHistoryOperationUnsupported</c>, the same as <see cref="IHistorianDataSource.ReadModifiedAsync"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IHistorianDataUpdater
|
||||
{
|
||||
/// <summary>Inserts, replaces or upserts samples for one tag.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="mode">Update mode.</param>
|
||||
/// <param name="values">Samples to apply.</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>
|
||||
/// One status code per value, in order: Insert gives <c>GoodEntryInserted</c> or <c>BadEntryExists</c>;
|
||||
/// Replace gives <c>GoodEntryReplaced</c> or <c>BadNoEntryExists</c>; Update gives
|
||||
/// <c>GoodEntryInserted</c> or <c>GoodEntryReplaced</c>; a null timestamp gives <c>BadInvalidTimestamp</c>.
|
||||
/// </returns>
|
||||
Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
|
||||
IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct);
|
||||
|
||||
/// <summary>Deletes raw samples in a time range.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="startUtc">Range start (UTC).</param>
|
||||
/// <param name="endUtc">Range end (UTC).</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns><c>Good</c>, or <c>BadNoData</c> when the range held nothing.</returns>
|
||||
Task<uint> DeleteRawAsync(string tag, DateTime startUtc, DateTime endUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct);
|
||||
|
||||
/// <summary>Deletes the samples at exact timestamps.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timesUtc">Timestamps to delete (UTC).</param>
|
||||
/// <param name="context">Who is making the change (recorded in the modification trail).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One status code per time, in order: <c>Good</c> or <c>BadNoEntryExists</c>.</returns>
|
||||
Task<IReadOnlyList<uint>> DeleteAtTimeAsync(string tag, IReadOnlyList<DateTime> timesUtc, HistoryUpdateContext context,
|
||||
CancellationToken ct);
|
||||
}
|
||||
@@ -1,35 +1,35 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// One numeric sample the continuous-historization recorder drains to the historian's SQL
|
||||
/// live-value write path. Carries the minimal payload that path can ingest: an optional UTC
|
||||
/// timestamp, the coerced numeric value, and an OPC-DA quality byte.
|
||||
/// One sample the continuous-historization recorder drains to the backend's live-value write path: an
|
||||
/// optional UTC timestamp, the typed value (any <see cref="HistorianValueKind"/> the backend supports — the
|
||||
/// Gateway backend takes numeric kinds only, SqlServer takes all of them), and the OPC UA status code the
|
||||
/// sample was recorded with.
|
||||
/// </summary>
|
||||
/// <param name="TimestampUtc">
|
||||
/// UTC source timestamp of the sample, or <c>null</c> to defer to the historian's server-stamped
|
||||
/// current-time path (the SQL writer uses <c>SYSDATETIME()</c> when the timestamp is absent).
|
||||
/// UTC source timestamp of the sample, or <c>null</c> to have the backend stamp the current time.
|
||||
/// </param>
|
||||
/// <param name="Value">The coerced numeric value (the SQL analog write path is numeric-only).</param>
|
||||
/// <param name="Quality">OPC-DA-derived quality code carried to the historian (192 = Good).</param>
|
||||
public readonly record struct HistorizationValue(DateTime? TimestampUtc, double Value, ushort Quality);
|
||||
/// <param name="Value">The typed sample value.</param>
|
||||
/// <param name="StatusCode">OPC UA status code of the sample (0 = Good).</param>
|
||||
public readonly record struct HistorizationValue(DateTime? TimestampUtc, HistorianValue Value, uint StatusCode);
|
||||
|
||||
/// <summary>
|
||||
/// Seam over the historian's live-value write path used by the continuous-historization recorder.
|
||||
/// Lives in the abstraction layer so the Runtime recorder depends on it without taking a hard
|
||||
/// reference on the gRPC gateway driver; the gateway driver supplies the concrete adapter
|
||||
/// (<c>GatewayHistorianValueWriter</c>).
|
||||
/// reference on a backend driver; the selected backend provider supplies the concrete writer
|
||||
/// (<c>GatewayHistorianValueWriter</c> or <c>SqlServerHistorianValueWriter</c>).
|
||||
/// </summary>
|
||||
public interface IHistorianValueWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes a batch of live values for a single tag through the historian's SQL live-write path.
|
||||
/// Writes a batch of live values for a single tag through the backend's live-write path.
|
||||
/// Implementations are expected to be non-throwing: a transport/gateway error is surfaced as a
|
||||
/// <c>false</c> result so the recorder retains the entries and retries, rather than as an
|
||||
/// exception.
|
||||
/// </summary>
|
||||
/// <param name="tag">Fully-qualified historian tag the values are recorded against.</param>
|
||||
/// <param name="values">The numeric samples to write, in append order.</param>
|
||||
/// <param name="values">The samples to write, in append order.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns><c>true</c> on a successful (or durably-queued) gateway ack; <c>false</c> on a retryable failure.</returns>
|
||||
/// <returns><c>true</c> when the backend stored (or durably queued) the batch; <c>false</c> on a retryable failure.</returns>
|
||||
Task<bool> WriteLiveValuesAsync(string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct);
|
||||
}
|
||||
|
||||
@@ -13,6 +13,13 @@ public interface IHistorizationOutbox : IDisposable
|
||||
/// <summary>Lifetime count of entries dropped because an append would have exceeded capacity.</summary>
|
||||
long DroppedCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Lifetime count of persisted records that could not be decoded and were skipped (never
|
||||
/// thrown, so a corrupt record cannot jam the drain). Defaults to 0 for implementations that
|
||||
/// cannot encounter unreadable records.
|
||||
/// </summary>
|
||||
long UnreadableCount => 0;
|
||||
|
||||
/// <summary>
|
||||
/// Appends <paramref name="entry"/> to the tail of the durable buffer.
|
||||
/// <para>
|
||||
|
||||
@@ -55,6 +55,40 @@ public sealed class NullHistorianDataSource : IHistorianDataSource
|
||||
int maxEvents,
|
||||
CancellationToken cancellationToken) => Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c> (Gitea #686): there is no history, so the empty result is the correct answer in either
|
||||
/// direction — no need for the server to walk back through forward reads to find nothing.</remarks>
|
||||
public bool SupportsReverseRaw => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>0</c>: no history, no ceiling (Gitea #694).</remarks>
|
||||
public int MaxRawValuesPerRead => 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><c>true</c>, like <see cref="SupportsReverseRaw"/> (Gitea #691.4 review; <c>false</c> under #675): there is no
|
||||
/// history to read in either direction — no need for the server to walk back through empty forward reads.</remarks>
|
||||
public bool SupportsReverseEvents => true;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The empty result in either direction.</remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName,
|
||||
DateTime startUtc,
|
||||
DateTime endUtc,
|
||||
int maxEvents,
|
||||
bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
Task.FromResult(EmptyEvents);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// EMPTY (Gitea #682): with no historian configured the server advertises no aggregate functions, and a processed
|
||||
/// HistoryRead answers <c>BadAggregateNotSupported</c> per node rather than <c>GoodNoData</c> — an aggregate is a
|
||||
/// statistic the server cannot compute without a history, and advertising the SDK's full list on a server with no
|
||||
/// historian misled capability discovery. Raw / AtTime / Events reads still return the empty <c>GoodNoData</c> result.
|
||||
/// </remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => HistoryAggregateTypes.None;
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => new(
|
||||
TotalQueries: 0,
|
||||
|
||||
@@ -107,11 +107,13 @@ public sealed record AlarmAcknowledgeRequest(
|
||||
/// watched happen: a reconnect snapshot entry (Galaxy) or the first event for a condition after a driver start or
|
||||
/// deploy respawn (OpcUaClient). It governs <b>history only, never node state</b>: the host still projects the
|
||||
/// state, but writes no <c>alerts</c> row when the re-statement changes nothing the condition shows (a Retrigger on
|
||||
/// an Active+unacked condition with the same message and severity; a Clear on an already-inactive condition) — or
|
||||
/// an Active+unacked condition with the same message and severity; a Clear on an already-inactive condition; an
|
||||
/// Acknowledge on an already-acknowledged condition) — or
|
||||
/// when a flagged <see cref="AlarmTransitionKind.Unspecified"/> update moves the message/severity of an
|
||||
/// already-active condition (node updated, one condition event, active/acked kept, no row). AbCip ALMA sets the
|
||||
/// flag for the last shape (the governing limit changed, no new occurrence); OpcUaClient's first sight of an
|
||||
/// inactive condition after a respawn can reach it too. <c>false</c> (the default, every source that does not
|
||||
/// flag for the last shape (the governing limit changed, no new occurrence). OpcUaClient (Gitea #657) re-states a
|
||||
/// first-seen inactive+acked condition as a flagged Clear then a flagged Acknowledge, so a host already showing
|
||||
/// that writes nothing and a host retaining it Active gets a real "Cleared". <c>false</c> (the default, every source that does not
|
||||
/// opt in) keeps every transition's row.
|
||||
/// </param>
|
||||
public sealed record AlarmEventArgs(
|
||||
@@ -147,6 +149,11 @@ public enum AlarmSeverity { Low, Medium, High, Critical }
|
||||
/// transition-kind enums (e.g. Galaxy's <c>GalaxyAlarmTransitionKind</c>).
|
||||
/// <see cref="ActiveAcknowledged"/> (Gitea #656) means "the condition is active AND acknowledged", applied by the
|
||||
/// host as ONE projection — never Raise-then-Acknowledge, which would show clients an Active+unacked intermediate.
|
||||
/// <see cref="InactiveUnacknowledged"/> (Gitea #657) is its mirror: "the condition is inactive AND unacknowledged"
|
||||
/// (it went active and cleared unacknowledged), applied as ONE projection — never Raise-then-Clear, which showed
|
||||
/// clients the condition Active again for an instant. The host writes rows only for what actually changed:
|
||||
/// "Activated" + "Cleared" when the condition was acknowledged (a new occurrence must have happened), "Cleared"
|
||||
/// alone when it was Active+unacked, none when it already showed inactive+unacked.
|
||||
/// Members are append-only: their numeric values must not move.
|
||||
/// </summary>
|
||||
public enum AlarmTransitionKind { Unspecified = 0, Raise, Acknowledge, Clear, Retrigger, ActiveAcknowledged }
|
||||
public enum AlarmTransitionKind { Unspecified = 0, Raise, Acknowledge, Clear, Retrigger, ActiveAcknowledged, InactiveUnacknowledged }
|
||||
|
||||
@@ -24,8 +24,9 @@ public interface IDriver
|
||||
|
||||
/// <summary>
|
||||
/// Apply a config change in place without tearing down the driver process.
|
||||
/// Invoked by the v2 <c>DriverInstanceActor</c> when ApplyDelta reports that only this
|
||||
/// driver's config changed in the new deployment.
|
||||
/// <para>The runtime does not call this today: since #516 a DriverConfig change is a stop + respawn,
|
||||
/// and #665 deleted <c>DriverInstanceActor</c>'s in-place <c>ApplyDelta</c> path. Kept as the in-place
|
||||
/// contract each driver implements and tests.</para>
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Per <c>docs/v2/driver-stability.md</c> §"In-process only (Tier A/B)" — Reinitialize is the
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Optional driver capability (Gitea #676) — a driver that can tell, from its protocol, that its connection is
|
||||
/// DEFINITIVELY gone (it can never recover in place, only a re-initialise brings it back) raises
|
||||
/// <see cref="ConnectionLost"/> so the host re-initialises it at once instead of waiting out the #527 comms-loss
|
||||
/// watchdog (~90 s of frozen <see cref="DriverHealth.LastSuccessfulRead"/>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Definitive only.</b> An AMBIGUOUS failure — a timeout, a stalled socket, "no communication" — must NOT
|
||||
/// raise: the connection may yet resume in place (a <c>docker pause</c>, a congested link), and tearing it
|
||||
/// down would cost the session, its subscriptions and the in-place recovery. Ambiguous failures keep going
|
||||
/// through the watchdog, whose conjunction (non-Healthy AND a frozen read stamp) is the honest test for them.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The host (<c>DriverInstanceActor</c>) acts on a raise only while the driver is Connected — in any other
|
||||
/// state a re-initialise is already pending or running — and it subscribes for its own lifetime, not per
|
||||
/// connect, because the <see cref="IDriver"/> instance can outlive the actor. A driver should raise at most
|
||||
/// once per connection and re-arm when it is re-initialised.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IDriverConnectionLossSource
|
||||
{
|
||||
/// <summary>Raised (on the driver's own thread) when the driver's connection is definitively lost.</summary>
|
||||
event EventHandler<DriverConnectionLossEventArgs>? ConnectionLost;
|
||||
}
|
||||
|
||||
/// <summary>Event payload for <see cref="IDriverConnectionLossSource.ConnectionLost"/>.</summary>
|
||||
/// <param name="Reason">Driver-supplied reason for the diagnostic log (e.g. the protocol status that proved the loss).</param>
|
||||
public sealed record DriverConnectionLossEventArgs(string Reason);
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
|
||||
/// <summary>Stable delivery GUIDs for persisted alarm queue rows.</summary>
|
||||
public static class AlarmDeliveryIds
|
||||
{
|
||||
/// <summary>
|
||||
/// Converts an existing durable row key into a nonzero version-8 GUID. Version 1 of this encoding hashes
|
||||
/// UTF-8 "OtOpcUa.AlarmDelivery/v1\0" followed by the exact persisted key with SHA-256, takes the first
|
||||
/// 16 bytes, sets the UUID version/variant bits, and interprets the bytes in network order.
|
||||
/// This encoding must remain stable across retries, upgrades and replicated failover.
|
||||
/// </summary>
|
||||
/// <param name="queueId">The persisted row key, never a hash recomputed from the current payload.</param>
|
||||
/// <returns>The row's stable delivery identity.</returns>
|
||||
public static Guid FromQueueId(string queueId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(queueId);
|
||||
var digest = SHA256.HashData(Encoding.UTF8.GetBytes("OtOpcUa.AlarmDelivery/v1\0" + queueId));
|
||||
digest[6] = (byte)((digest[6] & 0x0F) | 0x80);
|
||||
digest[8] = (byte)((digest[8] & 0x3F) | 0x80);
|
||||
return new Guid(digest.AsSpan(0, 16), bigEndian: true);
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
@@ -34,4 +35,31 @@ public sealed record AlarmHistorianEvent(
|
||||
string Message,
|
||||
string User,
|
||||
string? Comment,
|
||||
DateTime TimestampUtc);
|
||||
DateTime TimestampUtc)
|
||||
{
|
||||
/// <summary>
|
||||
/// Delivery identity, minted once for direct sends and replaced by the durable queue's identity when drained.
|
||||
/// Excluded from payload serialization and business equality so redundant enqueues still converge.
|
||||
/// A record copy preserves this identity; an explicitly empty identity is invalid for delivery.
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public Guid DeliveryId { get; init; } = Guid.NewGuid();
|
||||
|
||||
/// <summary>Compares the ten business fields, excluding transport delivery identity.</summary>
|
||||
/// <param name="other">The event to compare.</param>
|
||||
/// <returns>Whether the business fields are equal.</returns>
|
||||
public bool Equals(AlarmHistorianEvent? other) => ReferenceEquals(this, other) || other is not null
|
||||
&& AlarmId == other.AlarmId && EquipmentPath == other.EquipmentPath && AlarmName == other.AlarmName
|
||||
&& AlarmTypeName == other.AlarmTypeName && Severity == other.Severity && EventKind == other.EventKind
|
||||
&& Message == other.Message && User == other.User && Comment == other.Comment && TimestampUtc == other.TimestampUtc;
|
||||
|
||||
/// <summary>Hashes the same business fields used by equality, excluding delivery identity.</summary>
|
||||
/// <returns>The business-field hash code.</returns>
|
||||
public override int GetHashCode()
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(AlarmId); hash.Add(EquipmentPath); hash.Add(AlarmName); hash.Add(AlarmTypeName); hash.Add(Severity);
|
||||
hash.Add(EventKind); hash.Add(Message); hash.Add(User); hash.Add(Comment); hash.Add(TimestampUtc);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -603,6 +603,9 @@ public sealed class LocalDbStoreAndForwardSink : IAlarmHistorianSink, IDisposabl
|
||||
try
|
||||
{
|
||||
evt = JsonSerializer.Deserialize<AlarmHistorianEvent>(payload);
|
||||
// The durable key survives retry, payload edits, legacy migration and replicated failover.
|
||||
// Never derive delivery identity from a reserialized payload or an individual RPC attempt.
|
||||
if (evt is not null) evt = evt with { DeliveryId = AlarmDeliveryIds.FromQueueId(id) };
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
|
||||
@@ -221,7 +221,9 @@ public sealed class CipConnection : ICipChannel, IAsyncDisposable
|
||||
_o.RpiMicros, size, _route);
|
||||
var frame = ConnectionManagerServices.ForwardOpen(p);
|
||||
_diagnostics.CountForwardOpen();
|
||||
return CipReply.Parse(await client.SendRRDataAsync(frame, _o.Timeout, ct).ConfigureAwait(false));
|
||||
var raw = await client.SendRRDataAsync(frame, _o.Timeout, ct).ConfigureAwait(false);
|
||||
_diagnostics.NotifyExchange(frame, raw); // evidence tap: the granted RPIs / connection ids (#566 gate)
|
||||
return CipReply.Parse(raw);
|
||||
}
|
||||
|
||||
private ushort NextConnectionSerial()
|
||||
|
||||
@@ -92,8 +92,8 @@ internal sealed class ConnectionDiagnostics
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evidence tap for the gate harness (#582 / #572): sees the CIP request and the raw CIP reply of every
|
||||
/// completed exchange. Null in production; set only by the read-only differential harness. A throwing
|
||||
/// Evidence tap for the gate harness (#582 / #572 / #566): sees the CIP request and the raw CIP reply of every
|
||||
/// completed exchange, Forward Opens included (their replies carry the granted RPIs). Null in production; set only by the read-only differential harness. A throwing
|
||||
/// observer is swallowed so it can never affect a request.
|
||||
/// </summary>
|
||||
internal Action<ReadOnlyMemory<byte>, ReadOnlyMemory<byte>>? ExchangeObserver { get; set; }
|
||||
|
||||
@@ -360,8 +360,8 @@ public sealed class AbCipDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
// #630: InitializeAsync is reachable on a live driver (DriverInstanceActor.AdoptConfigDuringInit
|
||||
// runs it with no teardown first). Release the previous device states' probe loops + handles before
|
||||
// #630: InitializeAsync is reachable on a live driver with no teardown first (the IDriver contract
|
||||
// allows it, and a reconnect retry can overlap a running attempt). Release the previous device states' probe loops + handles before
|
||||
// the overwrite below. Deliberately NOT ShutdownAsync: that also drops _poll and the subscriptions.
|
||||
await ReleaseDeviceStatesAsync(cancellationToken).ConfigureAwait(false);
|
||||
_health = new DriverHealth(DriverState.Initializing, null, null);
|
||||
|
||||
@@ -145,8 +145,8 @@ public sealed class AbLegacyDriver : IDriver, IReadable, IWritable, ITagDiscover
|
||||
/// <inheritdoc />
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
// #630: InitializeAsync can run on a live driver (DriverInstanceActor.AdoptConfigDuringInit calls it
|
||||
// without a teardown first). Release the previous device states' probe loops + runtimes BEFORE the
|
||||
// #630: InitializeAsync can run on a live driver without a teardown first (the IDriver contract
|
||||
// allows it, and a reconnect retry can overlap a running attempt). Release the previous device states' probe loops + runtimes BEFORE the
|
||||
// overwrite below, or they are orphaned beyond the reach of ShutdownAsync. Deliberately NOT
|
||||
// ShutdownAsync: that also drops _poll and every subscription, which are owned by the shutdown path (the actor re-subscribes after every successful init).
|
||||
await ReleaseDeviceStatesAsync().ConfigureAwait(false);
|
||||
@@ -693,7 +693,7 @@ public sealed class AbLegacyDriver : IDriver, IReadable, IWritable, ITagDiscover
|
||||
LibplctagPlcAttribute: state.Profile.LibplctagPlcAttribute,
|
||||
// Normalised exactly like tag paths (PLC-5 octal I/O -> decimal); an unparseable probe address is passed through
|
||||
// so the transport reports the failure as it always did.
|
||||
TagName: AbLegacyAddress.TryParse(_options.Probe.ProbeAddress, state.Options.PlcFamily)?.ToLibplctagName()
|
||||
TagName: AbLegacyProbeAddress.Normalise(_options.Probe.ProbeAddress, state.Options.PlcFamily)
|
||||
?? _options.Probe.ProbeAddress!,
|
||||
Timeout: _options.Probe.Timeout);
|
||||
|
||||
|
||||
@@ -98,12 +98,18 @@ public sealed class AbLegacyDriverProbe : IDriverProbe
|
||||
?? FirstProbeableTagAddress(opts, firstDevice)
|
||||
?? "S:0";
|
||||
|
||||
// Normalise through the same helper as the driver's probe loop (#660): PLC-5 octal I/O -> decimal.
|
||||
// An unparseable address is a clean failure here — the factory would otherwise throw outside the try.
|
||||
var normalisedName = AbLegacyProbeAddress.Normalise(tagName, firstDevice?.PlcFamily ?? AbLegacyPlcFamily.Slc500);
|
||||
if (normalisedName is null)
|
||||
return new(false, $"Probe address '{tagName}' is not a valid AbLegacy address for this PLC family.", null);
|
||||
|
||||
var p = new AbLegacyTagCreateParams(
|
||||
Gateway: parsed.Gateway,
|
||||
Port: parsed.Port,
|
||||
CipPath: parsed.CipPath,
|
||||
LibplctagPlcAttribute: profile.LibplctagPlcAttribute,
|
||||
TagName: tagName,
|
||||
TagName: normalisedName,
|
||||
Timeout: timeout);
|
||||
|
||||
var factory = _tagFactory ?? (AbLegacyTransportResolver.Resolve(opts) == AbLegacyTransport.Native
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.PlcFamilies;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.AbLegacy;
|
||||
|
||||
/// <summary>
|
||||
/// The single authority that turns an operator-authored probe address into the name handed to the tag
|
||||
/// factory: family-aware parse (PLC-5 <c>I:</c>/<c>O:</c> octal indices become decimal) then
|
||||
/// <see cref="AbLegacyAddress.ToLibplctagName"/> — the same normalisation tag paths get. Shared by the
|
||||
/// driver's probe loop and the Test Connect probe (#660).
|
||||
/// </summary>
|
||||
internal static class AbLegacyProbeAddress
|
||||
{
|
||||
/// <summary>Returns the normalised name, or null when <paramref name="address"/> does not parse for the family.</summary>
|
||||
public static string? Normalise(string? address, AbLegacyPlcFamily family) =>
|
||||
AbLegacyAddress.TryParse(address, family)?.ToLibplctagName();
|
||||
}
|
||||
@@ -67,7 +67,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
|
||||
// Authored calc tags, keyed by RawPath (identity + evaluation id). Built at construction so
|
||||
// DependencyRefs is available BEFORE InitializeAsync (the host reads it when spawning the mux-adapter),
|
||||
// and REBUILT on every ReinitializeAsync (an in-place ApplyDelta re-binds an edited calc-tag set).
|
||||
// and REBUILT on every ReinitializeAsync / body-carrying InitializeAsync (an in-place re-bind of an edited calc-tag set).
|
||||
private readonly Dictionary<string, CalculationTagDefinition> _tagsByRawPath = new(StringComparer.Ordinal);
|
||||
|
||||
// rawPath (a dependency) → the calc tags that read it AND are change-triggered. Drives OnDependencyValue.
|
||||
@@ -139,7 +139,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// (Re)build the authored tag table (<see cref="_tagsByRawPath"/>), the change-trigger fan-out
|
||||
/// (<see cref="_changeDependents"/>), the dependency-ref interest set (<see cref="_dependencyRefs"/>),
|
||||
/// and the per-tag state map (<see cref="_stateByRawPath"/>) from <paramref name="config"/>. Called
|
||||
/// once at construction AND on every <see cref="ReinitializeAsync"/> so an in-place ApplyDelta — a
|
||||
/// once at construction AND on every <see cref="ReinitializeAsync"/> so an in-place config change — a
|
||||
/// calc-tag add/remove/edit, or a script-source edit — actually takes effect. Extracted so the ctor
|
||||
/// and reinit paths share ONE construction routine and cannot drift.
|
||||
/// <para>Per-tag <see cref="TagState"/> is preserved for a tag that survives the edit (cheap last-known
|
||||
@@ -200,7 +200,7 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
}
|
||||
|
||||
// Reinit config-bind options — mirrors the factory's bind (case-insensitive, enum-string, trailing-comma
|
||||
// tolerant) so an ApplyDelta payload re-binds identically to the spawn payload.
|
||||
// tolerant) so an in-place config payload re-binds identically to the spawn payload.
|
||||
private static readonly JsonSerializerOptions ReinitJsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
@@ -209,8 +209,8 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
Converters = { new JsonStringEnumConverter() },
|
||||
};
|
||||
|
||||
/// <summary>Re-bind the authored raw calc tags from a new merged DriverConfig JSON (the ApplyDelta
|
||||
/// payload). The ctor-fixed <see cref="CalculationEvaluator"/> timeout is kept — <c>RunTimeout</c> is bound
|
||||
/// <summary>Re-bind the authored raw calc tags from a new merged DriverConfig JSON (an in-place
|
||||
/// config payload). The ctor-fixed <see cref="CalculationEvaluator"/> timeout is kept — <c>RunTimeout</c> is bound
|
||||
/// at spawn (a change to it is out of this in-place path's scope). A parse failure logs and keeps the
|
||||
/// current tags so a malformed delta can never blank the driver.</summary>
|
||||
private CalculationDriverOptions ParseOptions(string driverConfigJson)
|
||||
@@ -239,6 +239,35 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
return new CalculationDriverOptions { RawTags = _options.RawTags, RunTimeout = _options.RunTimeout };
|
||||
}
|
||||
|
||||
/// <summary>Re-bind the tag table from <paramref name="driverConfigJson"/> — shared by
|
||||
/// <see cref="ReinitializeAsync"/> and (for a config with a body) <see cref="InitializeAsync"/> so the two
|
||||
/// adoption paths cannot drift. Timers must already be stopped.</summary>
|
||||
private void AdoptConfig(string driverConfigJson)
|
||||
{
|
||||
var newOptions = ParseOptions(driverConfigJson);
|
||||
lock (_lock)
|
||||
{
|
||||
// Queued evaluations were prepared against the OUTGOING tag table (old script source, old
|
||||
// dependency set), so they are stale the moment the rebuild lands — drop them rather than
|
||||
// publish one last pre-edit value after the delta was applied.
|
||||
_pending.Clear();
|
||||
RebuildTagTable(newOptions);
|
||||
}
|
||||
// Drop compiled scripts (IScriptCacheOwner contract — stops collectible ALCs accreting AND forces a
|
||||
// recompile from each tag's NEW ScriptSource).
|
||||
_evaluator.ClearCompiledScripts();
|
||||
}
|
||||
|
||||
/// <summary>True when the supplied DriverConfig JSON carries a real body. The bootstrapper always passes a
|
||||
/// populated document; unit tests pass <c>"{}"</c> to exercise lifecycle shape — those keep the
|
||||
/// constructor-supplied tags (an empty document would otherwise bind to an EMPTY tag table).</summary>
|
||||
private static bool HasConfigBody(string? driverConfigJson)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(driverConfigJson)) return false;
|
||||
var trimmed = driverConfigJson.Trim();
|
||||
return trimmed is not "{}" and not "[]";
|
||||
}
|
||||
|
||||
/// <summary>Minimal reinit-bind DTO — the calc driver has no backend config, only the injected raw tags
|
||||
/// (+ an optional eval-timeout, ignored on reinit since the evaluator is ctor-fixed). Mirrors the factory's
|
||||
/// <c>CalculationDriverConfigDto</c>.</summary>
|
||||
@@ -257,8 +286,17 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
public string DriverType => DriverTypeNames.Calculation;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #664: <c>InitializeAsync</c> takes a config document and the <see cref="IDriver"/> contract is that the
|
||||
/// driver serves it (the runtime's mid-connect adoption path was deleted in #665) — so it adopts a config carrying a
|
||||
/// real body (the #516 convention every other driver follows), and disposes any running timer scheduler
|
||||
/// before arming a fresh one (#630: re-initialise without teardown must not leak). A <c>"{}"</c> / empty
|
||||
/// document keeps the constructor-supplied tags, as unit tests rely on.
|
||||
/// </remarks>
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
StopTimers();
|
||||
if (HasConfigBody(driverConfigJson)) AdoptConfig(driverConfigJson);
|
||||
_running = true;
|
||||
StartTimers();
|
||||
_logger.LogInformation(
|
||||
@@ -270,27 +308,13 @@ public sealed class CalculationDriver : IDriver, IDependencyConsumer, ISubscriba
|
||||
/// <inheritdoc />
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
// A calc-tag add/remove/edit — or a script edit that changes the injected scriptSource — changes the
|
||||
// merged DriverConfig, which DriverSpawnPlanner routes to an in-place ApplyDelta → THIS call (a
|
||||
// respawn fires only on a DriverType/ResilienceConfig change). So the authored tag table is NOT
|
||||
// fixed at construction: we re-bind the new config and REBUILD the tag table + dependency-ref set
|
||||
// here. Without this, added tags never compute, removed tags keep publishing, and edited scripts
|
||||
// serve stale results (deploy reports success but nothing changed). After the rebuild _dependencyRefs
|
||||
// reflects the new authored tags, and the host re-registers the mux interest post-reinit (the
|
||||
// DriverInstanceActor.DeltaApplied → DriverHostActor path, sequenced AFTER this completes).
|
||||
// An in-place config change: re-bind the new config and REBUILD the tag table + dependency-ref set, so
|
||||
// added tags compute, removed tags stop publishing and edited scripts serve the new result. The runtime
|
||||
// itself applies a DriverConfig change by respawning (#516/#665), where a fresh driver registers its
|
||||
// new dependency refs on construction; this path keeps the IDriver contract honest for any caller.
|
||||
StopTimers();
|
||||
var newOptions = ParseOptions(driverConfigJson);
|
||||
lock (_lock)
|
||||
{
|
||||
// Queued evaluations were prepared against the OUTGOING tag table (old script source, old
|
||||
// dependency set), so they are stale the moment the rebuild lands — drop them rather than
|
||||
// publish one last pre-edit value after the delta was applied.
|
||||
_pending.Clear();
|
||||
RebuildTagTable(newOptions);
|
||||
}
|
||||
// Drop compiled scripts (IScriptCacheOwner contract — stops collectible ALCs accreting AND forces a
|
||||
// recompile from each tag's NEW ScriptSource) + re-arm timers off the rebuilt tag set.
|
||||
_evaluator.ClearCompiledScripts();
|
||||
AdoptConfig(driverConfigJson);
|
||||
// Re-arm timers off the rebuilt tag set.
|
||||
_running = true;
|
||||
StartTimers();
|
||||
return Task.CompletedTask;
|
||||
|
||||
@@ -196,8 +196,8 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
/// <returns>A completed task.</returns>
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
// #630: release the previous device states' loops + clients BEFORE they are overwritten (the actor's
|
||||
// AdoptConfigDuringInit calls this without a teardown first). Not ShutdownAsync: that also drops _poll
|
||||
// #630: release the previous device states' loops + clients BEFORE they are overwritten (a live driver can be
|
||||
// re-initialised without a teardown first). Not ShutdownAsync: that also drops _poll
|
||||
// and subscriptions, which are owned by the shutdown path (the actor re-subscribes after every successful init).
|
||||
await ReleaseDeviceStatesAsync().ConfigureAwait(false);
|
||||
ResetAlarmHealth();
|
||||
@@ -1232,13 +1232,16 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
ProgramInfo: programInfo is not null,
|
||||
Timers: timer is not null,
|
||||
Status: statusProbe is not null);
|
||||
state.LastStatus = statusProbe;
|
||||
|
||||
state.FixedTreeCache = new FocasFixedTreeCache(sys, [.. axes], [.. spindles], caps);
|
||||
// The bootstrap IS an answer from the device, and Identity comes entirely from it —
|
||||
// so stamp liveness here as well as per tick, or a CNC whose axis poll never
|
||||
// succeeds would serve a perfectly valid sysinfo as stale.
|
||||
state.LastFixedPollUtc = DateTime.UtcNow;
|
||||
// Published LAST (release write): LastStatus is the field callers watch for "the first
|
||||
// answer is in", and an authored FIXED:Status/* read is only servable once
|
||||
// FixedTreeCache and LastFixedPollUtc are set (TryReadFixedTreeOnDevice). Publishing it
|
||||
// first let a reader see a status while the read still returned BadCommunicationError (#673).
|
||||
state.LastStatus = statusProbe;
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; }
|
||||
catch (Exception ex)
|
||||
@@ -1998,7 +2001,12 @@ public sealed class FocasDriver : IDriver, IReadable, IWritable, ITagDiscovery,
|
||||
/// </summary>
|
||||
public ConcurrentDictionary<string, byte> CoalesceRejected { get; } = new(StringComparer.Ordinal);
|
||||
/// <summary>Gets or sets the last <c>cnc_statinfo</c> snapshot (#257); null until the first answer.</summary>
|
||||
public FocasStatus? LastStatus { get; set; }
|
||||
public FocasStatus? LastStatus
|
||||
{
|
||||
get => Volatile.Read(ref _lastStatus);
|
||||
set => Volatile.Write(ref _lastStatus, value);
|
||||
}
|
||||
private FocasStatus? _lastStatus;
|
||||
/// <summary>Gets or sets the last program information snapshot.</summary>
|
||||
public FocasProgramInfo? LastProgramInfo { get; set; }
|
||||
/// <summary>Gets or sets the cached first-axis dynamic snapshot — feeds Program/Number, /MainNumber, /Sequence.</summary>
|
||||
|
||||
@@ -348,9 +348,18 @@ public sealed class GalaxyDriver
|
||||
ReplayAsync(cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #664: <c>InitializeAsync</c> takes a config document, and the <see cref="IDriver"/> contract is that a
|
||||
/// driver serves the one it is handed (the runtime's own mid-connect adoption path was deleted in #665). The
|
||||
/// driver cannot hot-swap its options (they shape the gw session; see <see cref="ReinitializeAsync"/>), so a
|
||||
/// non-equivalent config with a real body is REFUSED here too rather than silently ignored — the driver
|
||||
/// would otherwise connect on its old endpoint while the deploy reported the change applied. A
|
||||
/// <c>"{}"</c> / empty document is the unit-test lifecycle shape and keeps the constructor options.
|
||||
/// </remarks>
|
||||
public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (HasConfigBody(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson);
|
||||
|
||||
// Tests inject seams via the internal ctor; production InitializeAsync builds
|
||||
// the gateway client + session + per-capability runtime components from
|
||||
@@ -650,48 +659,65 @@ public sealed class GalaxyDriver
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (!string.IsNullOrWhiteSpace(driverConfigJson))
|
||||
{
|
||||
// In-place config reapply. The driver does not currently support hot-swapping
|
||||
// GalaxyDriverOptions at runtime — changing the gateway endpoint, MxAccess
|
||||
// client name, or reconnect policy requires tearing down the gw session,
|
||||
// supervisor, event pump, and address space. The host stack handles that via
|
||||
// DriverInstance restart, so this method only accepts an equivalent config (no
|
||||
// meaningful change) and refreshes health; a non-equivalent reapply throws
|
||||
// NotSupportedException so the caller knows the change wasn't applied.
|
||||
//
|
||||
// Materialise the incoming config and compare against the live options. We
|
||||
// refuse any change that would require a session teardown rather than
|
||||
// pretending to apply it.
|
||||
GalaxyDriverOptions incoming;
|
||||
try
|
||||
{
|
||||
// Reuse the factory's parse pipeline so any missing-required-field
|
||||
// error surfaces with the same diagnostic text as InitializeAsync.
|
||||
var transient = GalaxyDriverFactoryExtensions.CreateInstance(_driverInstanceId, driverConfigJson);
|
||||
incoming = transient.Options;
|
||||
// The transient instance never started a runtime — disposing is cheap.
|
||||
transient.Dispose();
|
||||
}
|
||||
catch (Exception ex) when (ex is not NotSupportedException and not ObjectDisposedException)
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
$"GalaxyDriver.ReinitializeAsync could not parse the incoming DriverConfig JSON for '{_driverInstanceId}': {ex.Message}",
|
||||
ex);
|
||||
}
|
||||
|
||||
if (!OptionsAreEquivalent(_options, incoming))
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
"GalaxyDriver.ReinitializeAsync does not support hot-swapping driver options at runtime " +
|
||||
"(gateway endpoint, MxAccess client name, reconnect policy, etc.). Restart the DriverInstance " +
|
||||
"through the host stack to apply a config change.");
|
||||
}
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(driverConfigJson)) RefuseNonEquivalentConfig(driverConfigJson);
|
||||
_health = new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>True when the supplied DriverConfig JSON carries a real body (not empty / <c>"{}"</c> / <c>"[]"</c>).</summary>
|
||||
private static bool HasConfigBody(string? driverConfigJson)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(driverConfigJson)) return false;
|
||||
var trimmed = driverConfigJson.Trim();
|
||||
return trimmed is not "{}" and not "[]";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Throw <see cref="NotSupportedException"/> unless <paramref name="driverConfigJson"/> describes options
|
||||
/// runtime-equivalent to the live ones (see <see cref="OptionsAreEquivalent"/>). Shared by
|
||||
/// <see cref="ReinitializeAsync"/> and <see cref="InitializeAsync"/> (#664) so neither path can silently
|
||||
/// ignore a config change it cannot apply.
|
||||
/// </summary>
|
||||
private void RefuseNonEquivalentConfig(string driverConfigJson)
|
||||
{
|
||||
// In-place config reapply. The driver does not currently support hot-swapping
|
||||
// GalaxyDriverOptions at runtime — changing the gateway endpoint, MxAccess
|
||||
// client name, or reconnect policy requires tearing down the gw session,
|
||||
// supervisor, event pump, and address space. The host stack handles that via
|
||||
// DriverInstance restart, so this method only accepts an equivalent config (no
|
||||
// meaningful change) and refreshes health; a non-equivalent reapply throws
|
||||
// NotSupportedException so the caller knows the change wasn't applied.
|
||||
//
|
||||
// Materialise the incoming config and compare against the live options. We
|
||||
// refuse any change that would require a session teardown rather than
|
||||
// pretending to apply it.
|
||||
GalaxyDriverOptions incoming;
|
||||
try
|
||||
{
|
||||
// Reuse the factory's parse pipeline so any missing-required-field
|
||||
// error surfaces with the same diagnostic text as InitializeAsync.
|
||||
var transient = GalaxyDriverFactoryExtensions.CreateInstance(_driverInstanceId, driverConfigJson);
|
||||
incoming = transient.Options;
|
||||
// The transient instance never started a runtime — disposing is cheap.
|
||||
transient.Dispose();
|
||||
}
|
||||
catch (Exception ex) when (ex is not NotSupportedException and not ObjectDisposedException)
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
$"GalaxyDriver could not parse the incoming DriverConfig JSON for '{_driverInstanceId}': {ex.Message}",
|
||||
ex);
|
||||
}
|
||||
|
||||
if (!OptionsAreEquivalent(_options, incoming))
|
||||
{
|
||||
throw new NotSupportedException(
|
||||
"GalaxyDriver does not support hot-swapping driver options at runtime " +
|
||||
"(gateway endpoint, MxAccess client name, reconnect policy, etc.). Restart the DriverInstance " +
|
||||
"through the host stack to apply a config change.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compare two <see cref="GalaxyDriverOptions"/> for runtime equivalence — every
|
||||
/// field that drives gw session shape or reconnect behaviour must match. Compared
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Gitea #693: HistoryRead AtTime on the Gateway backend, served from RAW gateway reads and evaluated with the SQL backends'
|
||||
/// shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4) — as #681 did for the per-sample aggregates — so the Gateway
|
||||
/// answers exactly what SqlServer and QuestDB answer for the same stored samples. AVEVA's own interpolated answer (the
|
||||
/// gateway's <c>ReadAtTime</c>) is not used: live, it returned the previous value as Good at a stored NULL's own time,
|
||||
/// null + Bad at a Good sample right after a NULL, a drifted value (99.99999999944) at an exact stored 100, and no
|
||||
/// Interpolated bit.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>What each requested time needs.</b> <c>prev</c> = the latest stored sample at or before it, <c>next</c> = the
|
||||
/// first stored sample after it. <c>next</c> is only read when the evaluator uses it (a usable <c>prev</c> that is not
|
||||
/// an exact hit).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Read plan — O(1) reads for a dense batch, never two round-trips per timestamp by default.</b> The distinct
|
||||
/// requested times are walked in ascending order with ONE forward window read <c>[t − 1 ms, max + 1 ms]</c> of at most
|
||||
/// <see cref="ReadRows"/> rows, which yields every in-window neighbour. The sample before the first time comes from a
|
||||
/// walk-back (below), the sample after the last time from one small forward probe. A window read that fills its budget
|
||||
/// (dense data, widely spread times) resolves the times before its last row; the next unresolved time either
|
||||
/// continues from that row (when the rows in between are expected to fit one read) or jumps there with a read that
|
||||
/// starts a density-sized margin before it (so its <c>prev</c> is normally in the same read). Ten times one second
|
||||
/// apart on a 1 Hz tag cost 1 read (all exact hits) or 3 (interpolated); ten times an hour apart about one read each.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Walk-back</b> (the gateway reads forward only), in two phases. <i>Gallop:</i> segments <c>(lo, hi]</c> stepping
|
||||
/// back from <c>hi</c>, each one forward read, the first sized from the density the window read showed (else
|
||||
/// <see cref="InitialWalkWidth"/>), an empty one growing the next ×4; after two empty segments the whole remainder down
|
||||
/// to the epoch is asked once — a complete answer ends the walk, a FULL one seeds the bisection with its last row.
|
||||
/// <i>Bisect</i> (once a stored row is known below <c>hi</c> — a full segment's last row, or the last row of an earlier
|
||||
/// window read): read <c>(mid, hi]</c>; empty ⇒ <c>hi = mid</c>; full ⇒ the known row moves up to its last row;
|
||||
/// complete with rows ⇒ its last row is the answer. Right after a full read the next segment is aimed at about a
|
||||
/// quarter of a read at the density it showed (one try — an empty aimed segment falls back to halving), and below
|
||||
/// 2 ms the segment is the whole stretch from the known row (never a 100 ns step).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> One request may take at most <c>64 + 16 × distinct times</c> raw reads (<see cref="BaseReads"/>,
|
||||
/// <see cref="ReadsPerTime"/>); past it — or with a strict-gateway ceiling below <see cref="MinBudget"/> rows — the
|
||||
/// node fails <c>BadTooManyOperations</c> with a message naming the limit. A jump read (to a time far past the previous
|
||||
/// read) asks for only <see cref="JumpRows"/> rows, so a thousand times spread over ten years cost about a thousand
|
||||
/// small reads, not a thousand 1,024-row reads.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Wire shape.</b> Every request bound is a whole millisecond (AVEVA stores milliseconds and truncates request
|
||||
/// times); a row counts only strictly INSIDE its request — AVEVA's synthetic boundary row at the query start (the value
|
||||
/// in effect there, <c>Quality = 133</c>, or a NULL row) can never be taken for a stored sample, nor a possible row at
|
||||
/// the query end; the epoch and before hold nothing. A Bad row (a stored NULL arrives as value 0, <c>q=1 opc=0</c>,
|
||||
/// #690) is value-less. Every stream is drained, never abandoned (a cancelled gateway stream evicts its pooled
|
||||
/// historian session). Nothing is asked past <c>now + </c><see cref="RawAggregateReader.LookAheadHorizon"/> (the
|
||||
/// historian stores nothing further ahead, and the gateway's AVEVA may refuse a far-future end); a strict gateway's
|
||||
/// row ceiling (#691.3) is learned and honoured — a short read is ordinary paging here, never re-probed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values.</b> A gateway value is a Double or a String whatever the node's data type (AVEVA stores every historized
|
||||
/// type as a numeric analog — a Boolean as a 0/1 <c>Int2</c>). The node's type hint converts each row BEFORE the
|
||||
/// shared evaluator runs, so its own type rules apply exactly as on the SQL backends: Boolean (value ≠ 0) and String
|
||||
/// are stepped; an integer type is an Int64 — interpolated, rounded half away from zero — and the result is returned
|
||||
/// as the node's integer type; Float / Double (and no hint) interpolate linearly as Double.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class GatewayAtTimeReader
|
||||
{
|
||||
/// <summary>The row budget of a window read (the first, or one continuing from a full one) and a walk-back segment.</summary>
|
||||
internal const int ReadRows = 1024;
|
||||
|
||||
/// <summary>The row budget of a jump read (a window read restarting at a time far past the previous read's last row).</summary>
|
||||
internal const int JumpRows = 64;
|
||||
|
||||
/// <summary>The row budget of the forward probe for the sample after the last time (boundary row + ties + one).</summary>
|
||||
internal const int ProbeRows = 2 + RawAggregateReader.PreWindowAllowance;
|
||||
|
||||
/// <summary>The smallest row budget a read can work with (the boundary row, a tie, one sample beyond).</summary>
|
||||
internal const int MinBudget = 4;
|
||||
|
||||
/// <summary>Raw reads one request may always take …</summary>
|
||||
internal const int BaseReads = 64;
|
||||
|
||||
/// <summary>
|
||||
/// … plus this many per distinct requested time. A runaway guard, not a budget (<c>HistoryReadDeadline</c> bounds the
|
||||
/// wall clock): a time inside a gap of a bursty tag legitimately costs a jump read plus a bisection back across the gap
|
||||
/// (≈ 6 reads — a 1 h/day tag asked at noon on consecutive days), so 4 refused requests AVEVA's own AtTime served.
|
||||
/// </summary>
|
||||
internal const int ReadsPerTime = 16;
|
||||
|
||||
/// <summary>The first walk-back segment's width when the window read showed no density.</summary>
|
||||
internal static readonly TimeSpan InitialWalkWidth = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>The widest walk-back segment sized from density, before ×4 growth.</summary>
|
||||
private static readonly TimeSpan MaxSeededWidth = TimeSpan.FromDays(1);
|
||||
|
||||
private const long Ms = TimeSpan.TicksPerMillisecond;
|
||||
|
||||
private static readonly TimeSpan OneMs = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private static readonly DateTime Floor = RawAggregateReader.ProbeFloor;
|
||||
|
||||
/// <summary>One drained read: the rows strictly inside the request (ascending) and whether the stream ended short.</summary>
|
||||
private readonly record struct Read(List<StoredSample> Rows, bool Complete);
|
||||
|
||||
/// <summary>Evaluates AtTime for <paramref name="timestampsUtc"/> (same length and order, duplicates included).</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="timestampsUtc">Requested times (any kind; treated as UTC, as on the SQL backends).</param>
|
||||
/// <param name="now">Server timestamp, the beyond-now rule and the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation.</param>
|
||||
/// <param name="valueType">The node's scalar value type (the hinted <c>IHistorianDataSource.ReadAtTimeAsync</c>), or null.</param>
|
||||
/// <param name="maxReads">The request's raw-read cap; null = <see cref="BaseReads"/> + <see cref="ReadsPerTime"/> per distinct time.</param>
|
||||
/// <returns>One value per requested time, stamped with it.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">More than <see cref="ReadRows"/> rows within one millisecond, the
|
||||
/// read cap, or a strict gateway ceiling below <see cref="MinBudget"/>.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling,
|
||||
string tag, IReadOnlyList<DateTime> timestampsUtc, DateTime now, CancellationToken ct, TypeCode? valueType = null,
|
||||
int? maxReads = null)
|
||||
{
|
||||
var requested = timestampsUtc.Select(t => DateTime.SpecifyKind(t, DateTimeKind.Utc)).ToArray();
|
||||
if (requested.Length == 0) return [];
|
||||
var times = requested.Distinct().Order().ToArray();
|
||||
var ctx = new Ctx(client, ceiling, tag, ct, valueType, maxReads ?? BaseReads + ReadsPerTime * times.Length);
|
||||
var neighbours = await ResolveAsync(ctx, times, now).ConfigureAwait(false);
|
||||
var result = new DataValueSnapshot[requested.Length];
|
||||
for (var i = 0; i < requested.Length; i++)
|
||||
{
|
||||
var t = requested[i];
|
||||
var (prev, next) = neighbours[t.Ticks];
|
||||
// A DateTime node is stepped (as on the SQL backends): a usable non-numeric next sends the evaluator down its
|
||||
// stepped branch, with the next's own status.
|
||||
if (valueType == TypeCode.DateTime && next is { IsUsable: true } n)
|
||||
next = n with { Value = HistorianValue.FromString(string.Empty) };
|
||||
result[i] = ToNodeType(AtTimeEvaluator.Evaluate(t, prev, next, now, now), valueType);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>One request's read context: the client, the type hint, and the read count against its cap.</summary>
|
||||
private sealed class Ctx(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag, CancellationToken ct,
|
||||
TypeCode? valueType, int maxReads)
|
||||
{
|
||||
public IHistorianGatewayClient Client { get; } = client;
|
||||
public GatewayRowCeiling Ceiling { get; } = ceiling;
|
||||
public string Tag { get; } = tag;
|
||||
public CancellationToken Ct { get; } = ct;
|
||||
public TypeCode? ValueType { get; } = valueType;
|
||||
public int MaxReads { get; } = maxReads;
|
||||
public int Reads { get; set; }
|
||||
|
||||
/// <summary>The smallest budget a read was served with (a strict gateway's ceiling shows here).</summary>
|
||||
public int SmallestBudget { get; set; } = int.MaxValue;
|
||||
}
|
||||
|
||||
/// <summary>True for the integer type codes (evaluated as Int64, returned as the node's own type).</summary>
|
||||
private static bool IsInteger(TypeCode? t) =>
|
||||
t is TypeCode.SByte or TypeCode.Byte or TypeCode.Int16 or TypeCode.UInt16 or TypeCode.Int32 or TypeCode.UInt32
|
||||
or TypeCode.Int64 or TypeCode.UInt64;
|
||||
|
||||
/// <summary>
|
||||
/// An evaluated Int64 (integer hint) returned as the node's integer type. A value the type cannot hold (only possible
|
||||
/// for a stored value already outside the node's range) stays the evaluator's Int64.
|
||||
/// </summary>
|
||||
private static DataValueSnapshot ToNodeType(DataValueSnapshot v, TypeCode? valueType)
|
||||
{
|
||||
if (v.Value is not long l || !IsInteger(valueType) || valueType == TypeCode.Int64) return v;
|
||||
try
|
||||
{
|
||||
return v with { Value = Convert.ChangeType(l, valueType!.Value, System.Globalization.CultureInfo.InvariantCulture) };
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
return v;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>prev / next for every time of <paramref name="times"/> (distinct, ascending).</summary>
|
||||
private static async Task<Dictionary<long, (StoredSample? Prev, StoredSample? Next)>> ResolveAsync(
|
||||
Ctx ctx, DateTime[] times, DateTime now)
|
||||
{
|
||||
var result = new Dictionary<long, (StoredSample?, StoredSample?)>(times.Length);
|
||||
var horizon = MsFloor(DateTime.MaxValue - now > RawAggregateReader.LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + RawAggregateReader.LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue.AddTicks(-Ms));
|
||||
|
||||
// Nothing is stored at or before the epoch.
|
||||
var i = 0;
|
||||
for (; i < times.Length && times[i] <= Floor; i++) result[times[i].Ticks] = (null, null);
|
||||
if (i == times.Length) return result;
|
||||
|
||||
// Reads are planned on times clamped to the horizon (nothing is stored beyond it, and no read ends past it).
|
||||
var qe = Min(MsFloor(Min(times[^1], horizon)) + OneMs, horizon);
|
||||
var qs = RequestStart(Min(times[i], horizon));
|
||||
var budget = ReadRows;
|
||||
StoredSample? before = null; // the latest stored sample at or before qs …
|
||||
var beforeKnown = false; // … once known
|
||||
DateTime? walkKnown = null; // a stored sample exists here (at or before qs): the walk-back bisects above it
|
||||
var walkWidth = InitialWalkWidth;
|
||||
var probeFrom = -1; // first index whose next lies at or after qe (complete last read)
|
||||
|
||||
while (true)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, qe, budget).ConfigureAwait(false);
|
||||
var rows = read.Rows;
|
||||
if (!read.Complete && rows.Count == 0)
|
||||
throw TooDense(ctx, qs);
|
||||
if (rows.Count >= 2 && !beforeKnown && walkKnown is null)
|
||||
walkWidth = Clamp(TimeSpan.FromTicks(4 * (rows[^1].TimeUtc - rows[0].TimeUtc).Ticks / (rows.Count - 1)));
|
||||
|
||||
// A short stream saw every row up to qe; a full one every row before its last row's time x.
|
||||
var x = read.Complete ? qe : rows[^1].TimeUtc;
|
||||
var j = -1; // index of the last row at or before the current time
|
||||
var k = i;
|
||||
for (; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var tr = Min(t, horizon);
|
||||
if (!read.Complete && tr >= x) break;
|
||||
while (j + 1 < rows.Count && rows[j + 1].TimeUtc <= tr) j++;
|
||||
|
||||
StoredSample? prev;
|
||||
if (j >= 0) prev = rows[j];
|
||||
else
|
||||
{
|
||||
if (!beforeKnown)
|
||||
{
|
||||
// #696: prove the future stretch empty before jumping to now. Historian clocks or imports
|
||||
// can place real rows there; a full bounded probe seeds the original backward search.
|
||||
var searchEnd = qs;
|
||||
if (qs > now && walkKnown is null)
|
||||
{
|
||||
var future = await ReadTrustedAsync(ctx, MsFloor(now),
|
||||
Min(MsFloor(qs) + OneMs, horizon), ReadRows).ConfigureAwait(false);
|
||||
StoredSample? latest = null;
|
||||
foreach (var row in future.Rows)
|
||||
if (row.TimeUtc <= qs) latest = row;
|
||||
if (future.Complete)
|
||||
{
|
||||
before = latest;
|
||||
beforeKnown = latest is not null;
|
||||
searchEnd = now;
|
||||
}
|
||||
else if (latest is { } found)
|
||||
walkKnown = found.TimeUtc;
|
||||
}
|
||||
if (!beforeKnown)
|
||||
before = await WalkBackAsync(ctx, searchEnd, walkKnown, walkWidth).ConfigureAwait(false);
|
||||
beforeKnown = true;
|
||||
}
|
||||
prev = before;
|
||||
}
|
||||
|
||||
StoredSample? next = null;
|
||||
if (NeedsNext(t, prev) && t < horizon)
|
||||
{
|
||||
if (j + 1 < rows.Count) next = rows[j + 1];
|
||||
else if (probeFrom < 0) probeFrom = k; // after every row of a complete read: the probe answers it
|
||||
}
|
||||
result[t.Ticks] = (prev, next);
|
||||
}
|
||||
|
||||
if (k == times.Length) break;
|
||||
|
||||
// Truncated before times[k]: continue from the last row, or jump to times[k] with a density-sized margin and a
|
||||
// small budget (it needs that time's neighbours, and whatever later times follow closely).
|
||||
var tn = Min(times[k], horizon);
|
||||
var spanTicks = Math.Max(1, (x - rows[0].TimeUtc).Ticks);
|
||||
var density = (double)rows.Count / spanTicks; // rows per tick
|
||||
var gapRows = density * (tn - x).Ticks;
|
||||
var margin = Clamp(TimeSpan.FromTicks((long)Math.Min(TimeSpan.MaxValue.Ticks / 2.0, JumpRows / 4.0 / Math.Max(density, 1e-18))));
|
||||
var resume = MsFloor(x) - OneMs;
|
||||
var jump = gapRows >= ReadRows / 2.0;
|
||||
var nextQs = jump ? Max(RequestStart(tn - Min(margin, tn - Floor)), resume) : resume;
|
||||
if (nextQs <= qs) throw TooDense(ctx, x);
|
||||
|
||||
if (nextQs < x)
|
||||
{
|
||||
// Every row before x was seen: the latest at or before nextQs is known.
|
||||
var lastIndex = rows.FindLastIndex(r => r.TimeUtc <= nextQs);
|
||||
if (lastIndex >= 0)
|
||||
{
|
||||
before = rows[lastIndex];
|
||||
beforeKnown = true;
|
||||
}
|
||||
walkKnown = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
beforeKnown = false;
|
||||
before = null;
|
||||
walkKnown = x;
|
||||
walkWidth = margin;
|
||||
}
|
||||
budget = jump ? JumpRows : ReadRows;
|
||||
qs = nextQs;
|
||||
i = k;
|
||||
}
|
||||
|
||||
if (probeFrom >= 0 && qe < horizon)
|
||||
{
|
||||
var after = await FirstAfterAsync(ctx, qe - OneMs, horizon, Min(times[^1], horizon)).ConfigureAwait(false);
|
||||
if (after is not null)
|
||||
for (var k = probeFrom; k < times.Length; k++)
|
||||
{
|
||||
var t = times[k];
|
||||
var (prev, next) = result[t.Ticks];
|
||||
if (next is null && t < horizon && NeedsNext(t, prev) && after.Value.TimeUtc > t)
|
||||
result[t.Ticks] = (prev, after);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>The evaluator reads <c>next</c> only for a usable <c>prev</c> that is not an exact hit.</summary>
|
||||
private static bool NeedsNext(DateTime t, StoredSample? prev) => prev is { IsUsable: true } p && p.TimeUtc != t;
|
||||
|
||||
/// <summary>One walk-back segment <c>(lo, hi]</c>: its oldest and newest rows, and whether the read was complete.</summary>
|
||||
private static async Task<(StoredSample? First, StoredSample? Last, bool Complete)> SegmentAsync(
|
||||
Ctx ctx, DateTime lo, DateTime hi)
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, MsFloor(lo), MsFloor(hi) + OneMs, ReadRows).ConfigureAwait(false);
|
||||
StoredSample? first = null, last = null;
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > lo && r.TimeUtc <= hi)
|
||||
{
|
||||
first ??= r;
|
||||
last = r;
|
||||
}
|
||||
return (first, last, read.Complete);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest stored sample with time in <c>(Floor, hi]</c> (see the type remarks: gallop, then bisect).
|
||||
/// <paramref name="known"/>: a stored sample exists at that time (≤ <paramref name="hi"/>) — the walk then bisects
|
||||
/// above it straight away, its first segment aimed at <paramref name="width"/>.
|
||||
/// </summary>
|
||||
private static async Task<StoredSample?> WalkBackAsync(Ctx ctx, DateTime hi, DateTime? known, TimeSpan width)
|
||||
{
|
||||
TimeSpan? aim = known is null ? null : width;
|
||||
|
||||
// Gallop: no stored row known below hi yet.
|
||||
var empties = 0;
|
||||
while (known is null)
|
||||
{
|
||||
if (hi <= Floor) return null;
|
||||
var rest = empties >= 2;
|
||||
var lo = rest || (hi - Floor).Ticks <= width.Ticks ? Floor : hi - width;
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo <= Floor) return null;
|
||||
hi = lo;
|
||||
empties++;
|
||||
width = Grow(width);
|
||||
continue;
|
||||
}
|
||||
// Full — a whole-remainder read of deep history too: its last row is a stored row below hi. Bisect above it.
|
||||
if (seg.Last is not { } seen) throw TooDense(ctx, lo);
|
||||
known = seen.TimeUtc;
|
||||
aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
|
||||
// Bisect: a stored row exists at `known`, nothing newer than `hi` matters; the answer is the newest row in [known, hi].
|
||||
var aimMissed = false;
|
||||
while (true)
|
||||
{
|
||||
var k = known.Value;
|
||||
DateTime lo;
|
||||
var aimed = false;
|
||||
if (hi - k < TimeSpan.FromMilliseconds(2))
|
||||
lo = k.AddTicks(-1); // the whole stretch from the known row: never a sub-millisecond step
|
||||
else
|
||||
{
|
||||
lo = k + TimeSpan.FromTicks((hi - k).Ticks / 2);
|
||||
if (aim is { } a && hi - a > lo)
|
||||
{
|
||||
lo = hi - a; // an aimed try right after a full read — until one comes back empty (a gap: halve from then on)
|
||||
aimed = true;
|
||||
}
|
||||
}
|
||||
aim = null;
|
||||
|
||||
var seg = await SegmentAsync(ctx, lo, hi).ConfigureAwait(false);
|
||||
if (seg.Complete)
|
||||
{
|
||||
if (seg.Last is { } found) return found;
|
||||
if (lo < k) return null; // the known row is gone (deleted meanwhile): nothing is left to find
|
||||
hi = lo;
|
||||
aimMissed |= aimed;
|
||||
continue;
|
||||
}
|
||||
if (seg.Last is not { } seen || seen.TimeUtc <= k) throw TooDense(ctx, k);
|
||||
known = seen.TimeUtc;
|
||||
if (!aimMissed) aim = AimFrom(seg.First!.Value.TimeUtc, seen.TimeUtc);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>About a quarter of a read at the density a full read showed (its rows from <paramref name="first"/> to <paramref name="last"/>).</summary>
|
||||
private static TimeSpan AimFrom(DateTime first, DateTime last) =>
|
||||
TimeSpan.FromTicks(Math.Max(Ms, (last - first).Ticks / 4));
|
||||
|
||||
/// <summary>The first stored sample after <paramref name="after"/> in <c>(qs, end)</c>, from a small forward read.</summary>
|
||||
private static async Task<StoredSample?> FirstAfterAsync(Ctx ctx, DateTime qs, DateTime end, DateTime after)
|
||||
{
|
||||
foreach (var budget in new[] { ProbeRows, ReadRows })
|
||||
{
|
||||
var read = await ReadTrustedAsync(ctx, qs, end, budget).ConfigureAwait(false);
|
||||
foreach (var r in read.Rows)
|
||||
if (r.TimeUtc > after) return r;
|
||||
if (read.Complete) return null;
|
||||
}
|
||||
throw TooDense(ctx, qs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One forward read <c>[qs, qe]</c> (whole milliseconds), drained, keeping only rows strictly inside it and after the
|
||||
/// epoch — counted against the request's read cap. A strict gateway's ceiling is learned on refusal and then asked for
|
||||
/// directly; a full read is not re-probed (the caller pages on, it does not need the rest of this read).
|
||||
/// </summary>
|
||||
private static async Task<Read> ReadTrustedAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
if (++ctx.Reads > ctx.MaxReads)
|
||||
throw new HistoryReadLimitExceededException(ctx.MaxReads,
|
||||
$"The AtTime read of '{ctx.Tag}' needed more than {ctx.MaxReads} raw reads ({BaseReads} + {ReadsPerTime} per " +
|
||||
"requested time)" + CeilingNote(ctx) + "; the Gateway backend serves AtTime from raw reads — request fewer times.");
|
||||
var (read, used) = await ctx.Ceiling.ReadAsync(budget, b => PassAsync(ctx, qs, qe, b), (_, _) => false)
|
||||
.ConfigureAwait(false);
|
||||
if (used < ctx.SmallestBudget) ctx.SmallestBudget = used;
|
||||
if (used < MinBudget)
|
||||
throw new HistoryReadLimitExceededException(used,
|
||||
$"The HistorianGateway's row ceiling ({used}, its Historian:ReadLimits:MaxValuesCeiling with " +
|
||||
$"AllowStreamingBeyondCeiling=false) is below the {MinBudget} rows an AtTime raw read of '{ctx.Tag}' needs.");
|
||||
return read;
|
||||
}
|
||||
|
||||
private static string CeilingNote(Ctx ctx) =>
|
||||
ctx.SmallestBudget < JumpRows
|
||||
? $", with reads bounded by the HistorianGateway's row ceiling of {ctx.SmallestBudget} (Historian:ReadLimits:MaxValuesCeiling)"
|
||||
: string.Empty;
|
||||
|
||||
private static async Task<Read> PassAsync(Ctx ctx, DateTime qs, DateTime qe, int budget)
|
||||
{
|
||||
var rows = new List<StoredSample>();
|
||||
var received = 0;
|
||||
await foreach (var raw in ctx.Client.ReadRawAsync(ctx.Tag, qs, qe, budget, ctx.Ct).ConfigureAwait(false))
|
||||
{
|
||||
received++; // drained to completion, never broken out of
|
||||
if (raw.Timestamp is null) continue;
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > qs && t < qe && t > Floor) rows.Add(ToStored(raw, t, ctx.ValueType));
|
||||
}
|
||||
return new Read(rows, received < budget);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A gateway row as a stored sample of the node's type (see the type remarks); a Bad row carries no value (#690: a
|
||||
/// stored NULL arrives as value 0).
|
||||
/// </summary>
|
||||
private static StoredSample ToStored(HistorianSample raw, DateTime t, TypeCode? valueType)
|
||||
{
|
||||
var status = SampleMapper.ToStatus(raw);
|
||||
var value = StatusBits.IsBad(status) ? HistorianValue.Empty
|
||||
: raw.HasNumericValue ? Typed(raw.NumericValue, valueType)
|
||||
: raw.HasStringValue ? HistorianValue.FromString(raw.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(t, value, status);
|
||||
}
|
||||
|
||||
/// <summary>A gateway number as the hinted kind, so the shared evaluator's own type branch decides stepped vs linear.</summary>
|
||||
private static HistorianValue Typed(double v, TypeCode? valueType) => valueType switch
|
||||
{
|
||||
TypeCode.Boolean => HistorianValue.FromBoolean(v != 0),
|
||||
TypeCode.String => HistorianValue.FromString(v.ToString("R", System.Globalization.CultureInfo.InvariantCulture)),
|
||||
// In Int64 range only (2^63 is exactly representable); NaN, ±∞ or out of range is no integer at all — Empty, which
|
||||
// StoredSample.FromStored makes BadNoData (unusable), never a saturated Good value.
|
||||
_ when IsInteger(valueType) => v >= -9.2233720368547758E18 && v < 9.2233720368547758E18
|
||||
? HistorianValue.FromInt64((long)Math.Round(v, MidpointRounding.AwayFromZero))
|
||||
: HistorianValue.Empty,
|
||||
_ => HistorianValue.FromDouble(v),
|
||||
};
|
||||
|
||||
/// <summary>The window read's start for a time: a millisecond before its millisecond (never before the epoch).</summary>
|
||||
private static DateTime RequestStart(DateTime t)
|
||||
{
|
||||
var s = MsFloor(t) - TimeSpan.FromMilliseconds(1);
|
||||
return s < Floor ? Floor : s;
|
||||
}
|
||||
|
||||
private static DateTime MsFloor(DateTime t) => new(t.Ticks - t.Ticks % Ms, DateTimeKind.Utc);
|
||||
|
||||
private static DateTime Min(DateTime a, DateTime b) => a <= b ? a : b;
|
||||
|
||||
private static DateTime Max(DateTime a, DateTime b) => a >= b ? a : b;
|
||||
|
||||
private static TimeSpan Min(TimeSpan a, TimeSpan b) => a <= b ? a : b;
|
||||
|
||||
private static TimeSpan Clamp(TimeSpan w) =>
|
||||
w < TimeSpan.FromMilliseconds(1) ? TimeSpan.FromMilliseconds(1) : w > MaxSeededWidth ? MaxSeededWidth : w;
|
||||
|
||||
private static TimeSpan Grow(TimeSpan w) => w.Ticks > TimeSpan.MaxValue.Ticks / 4 ? TimeSpan.MaxValue : w * 4;
|
||||
|
||||
private static HistoryReadLimitExceededException TooDense(Ctx ctx, DateTime at)
|
||||
{
|
||||
var rows = Math.Min(ReadRows, ctx.SmallestBudget);
|
||||
return new(rows,
|
||||
$"The AtTime read of '{ctx.Tag}' met more than {rows} stored samples within about one millisecond of {at:O}" +
|
||||
(rows < ReadRows ? $" (reads are bounded by the HistorianGateway's row ceiling of {rows})" : string.Empty) +
|
||||
"; the Gateway backend serves AtTime from raw reads.");
|
||||
}
|
||||
}
|
||||
+487
-20
@@ -1,7 +1,10 @@
|
||||
using System.Collections.Frozen;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
@@ -23,8 +26,50 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </remarks>
|
||||
public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The processed (aggregate) functions the Gateway backend serves — the single source of truth for this backend's
|
||||
/// aggregate set (#681): the per-sample aggregates evaluated from raw reads (every member of the
|
||||
/// <see cref="AggregateFamily.Simple"/> / <see cref="AggregateFamily.StartEnd"/> /
|
||||
/// <see cref="AggregateFamily.ActualTime"/> families of <see cref="AggregateFamilies"/>: Average, Count, Minimum,
|
||||
/// Maximum, Range, Start, End, Delta, MinimumActualTime, MaximumActualTime) plus the two answered natively by AVEVA
|
||||
/// (TimeAverage, Total — <see cref="AggregateModeMapper"/>). Any other aggregate throws
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> without calling the gateway. It is what
|
||||
/// <see cref="SupportedAggregates"/> returns, so the advertised <c>AggregateFunctions</c> folders and the server's
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too (#682).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
.Where(a => RawAggregateReader.Serves(a) || AggregateModeMapper.NativeAggregates.Contains(a))
|
||||
.ToFrozenSet();
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayHistorianDataSource> _logger;
|
||||
private readonly int _maxRawRows;
|
||||
private readonly TimeProvider _time;
|
||||
private readonly bool _atTimeFromRaw;
|
||||
|
||||
/// <summary>#693: AtTime from raw reads (true) or AVEVA's own interpolation (the kill switch).</summary>
|
||||
internal bool AtTimeFromRaw => _atTimeFromRaw;
|
||||
|
||||
/// <summary>The most #687 crowding re-reads one raw read makes (#694 F4): each at least doubles the pre-window room.</summary>
|
||||
private const int MaxCrowdedReReads = 8;
|
||||
|
||||
/// <summary>The default of <c>maxPreWindowRows</c>: <c>ServerHistorian:MaxTieClusterOverfetch</c>'s default.</summary>
|
||||
public const int DefaultMaxPreWindowRows = 65536;
|
||||
|
||||
/// <summary>
|
||||
/// The most rows before a raw window (inside the 1 ms look-back) a crowding re-read makes room for (#694 R2) — the
|
||||
/// server's tie-cluster limit (<c>ServerHistorian:MaxTieClusterOverfetch</c>), which every other tie path treats as
|
||||
/// the limit. More ⇒ <see cref="HistoryReadLimitExceededException"/> with <c>PreWindowRows</c> set.
|
||||
/// </summary>
|
||||
private readonly int _maxPreWindowRows;
|
||||
|
||||
/// <summary>The tie-cluster limit the crowding re-reads honour (#694 R2) — exposed so the factory wiring is testable.</summary>
|
||||
internal int MaxPreWindowRows => _maxPreWindowRows;
|
||||
|
||||
/// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
|
||||
private readonly GatewayRowCeiling _ceiling;
|
||||
private readonly GatewayTagPresenceCache _tags;
|
||||
|
||||
private readonly object _healthLock = new();
|
||||
private DateTime? _lastSuccessUtc;
|
||||
@@ -38,79 +83,440 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// <summary>Creates a gateway-backed historian data source.</summary>
|
||||
/// <param name="client">The gateway client seam used for all reads.</param>
|
||||
/// <param name="logger">Diagnostic logger; failures are recorded without leaking tag/host detail.</param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger)
|
||||
/// <param name="maxRawRows">
|
||||
/// In-window raw-row cap of one raw-evaluated processed read (<c>ServerHistorian:MaxTimeWeightedRawRows</c>;
|
||||
/// <= 0 = unbounded). Past it the node fails <c>BadTooManyOperations</c>.
|
||||
/// </param>
|
||||
/// <param name="timeProvider">Clock for result server timestamps and the raw look-ahead horizon; null = system.</param>
|
||||
/// <param name="atTimeFromRaw">
|
||||
/// #693: serve AtTime from raw reads with the shared AtTime rules (<see cref="GatewayAtTimeReader"/>; default);
|
||||
/// <c>false</c> = AVEVA's own interpolation passed through (<c>ServerHistorian:GatewayAtTimeFromRaw</c> kill switch).
|
||||
/// </param>
|
||||
/// <param name="maxPreWindowRows">
|
||||
/// The most rows before a raw window a crowding re-read makes room for (#694 R2; <c>ServerHistorian:MaxTieClusterOverfetch</c>,
|
||||
/// values below 1 count as 1). More ⇒ <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>).
|
||||
/// </param>
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows)
|
||||
{
|
||||
_maxPreWindowRows = Math.Max(1, maxPreWindowRows);
|
||||
_atTimeFromRaw = atTimeFromRaw;
|
||||
ArgumentNullException.ThrowIfNull(client);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
_maxRawRows = maxRawRows;
|
||||
_time = timeProvider ?? TimeProvider.System;
|
||||
_ceiling = new GatewayRowCeiling(_time);
|
||||
_tags = new GatewayTagPresenceCache(_client, _time);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse window (<c>start > end</c>) is refused with <see cref="NotSupportedException"/> before anything is sent
|
||||
/// and is not recorded as a health failure (Gitea #686): the gateway rejects it <c>InvalidArgument</c>, and the server
|
||||
/// never asks — <see cref="SupportsReverseRaw"/> is <c>false</c>, so it walks a reverse read back with forward reads.
|
||||
/// Stored samples only (#687 — Part 11 Raw): AVEVA's synthetic start-boundary row is never returned, see
|
||||
/// <see cref="ReadRawWindowAsync"/>.
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (startUtc > endUtc)
|
||||
throw new NotSupportedException("The HistorianGateway reads raw history oldest-first only (start > end).");
|
||||
|
||||
try
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var maxValues = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
var samples = new List<HistorianSample>();
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, startUtc, endUtc, maxValues, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
samples.Add(sample);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult([], ContinuationPoint: null);
|
||||
}
|
||||
|
||||
var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
var snapshots = SampleMapper.ToSnapshots(samples);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // the caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// An unbounded read over Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false: a
|
||||
// client-request limit (BadTooManyOperations), not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(maxValuesPerNode,
|
||||
$"The HistorianGateway refused the row budget of the raw read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); set NumValuesPerNode or narrow the window.");
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (#687: the pre-window margin swallowed the row budget), not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The stored samples of <c>[startUtc, endUtc)</c> (a point read <c>start == end</c>: of that instant), oldest first,
|
||||
/// at most <paramref name="maxValuesPerNode"/> (<c>0</c> = all) — without AVEVA's synthetic rows (#687).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// An AVEVA <c>Full</c> raw read starts with a row stamped exactly at the query start: the value in effect there
|
||||
/// (<c>Quality = 133</c>, the prior sample's value — never stored at that time) or, before the tag's first sample,
|
||||
/// a NULL row (value 0, Bad). Part 11 Raw with <c>ReturnBounds = false</c> wants stored values only, and the
|
||||
/// server's paging (#669) counts every returned row toward the page and resumes at the last returned timestamp,
|
||||
/// so a fabricated row would also be counted as a sample and re-fabricated at every resume point. The read
|
||||
/// therefore uses the same request start as the raw-evaluated aggregates (#681): it is requested from
|
||||
/// <see cref="RawAggregateReader.RequestStartFor"/> (1 ms early) and only rows
|
||||
/// <see cref="RawAggregateReader.IsWindowSample"/> are kept — the decision is by TIMESTAMP, because a stored NULL
|
||||
/// sample and the no-data boundary row look the same on the wire (both carry a value 0 with a Bad quality). The
|
||||
/// forward end is exclusive (the interface contract; the gateway's own end is inclusive), a point read inclusive.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Row budget.</b> <c>0</c> ("all values") asks for <see cref="int.MaxValue"/> (the client package and the
|
||||
/// gateway reject <c>max_values <= 0</c>, #686). A cap asks for <c>cap + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
|
||||
/// clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
|
||||
/// every page size within the allowance of the ceiling come back short. A strict gateway
|
||||
/// (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
|
||||
/// then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
|
||||
/// of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
|
||||
/// whole ceiling without reaching the window end, or a page short of its cap) fails
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
|
||||
/// paging. A read is complete when the stream came back short, filled the cap, or reached the window end. If rows
|
||||
/// BEFORE the window (the boundary row and stored samples inside the 1 ms margin, e.g. a tie cluster at
|
||||
/// <c>start − 1 ms</c>) used up the budget, the window is read ONCE more asking for <c>cap + m + </c>
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> (m = the pre-window rows the first read saw); only if that
|
||||
/// re-read is still short because of pre-window rows does the read fail with
|
||||
/// <see cref="HistoryReadLimitExceededException"/>. Every stream is consumed to completion (never broken out of —
|
||||
/// a cancelled gateway stream evicts its pooled historian session). <c>ReturnBounds</c> is not plumbed to the
|
||||
/// data source on any backend, so bounding values are never synthesised here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task<List<HistorianSample>> ReadRawWindowAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
|
||||
var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
|
||||
if (cap == 0)
|
||||
{
|
||||
var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
|
||||
if (all.Result.Complete) return all.Result.Samples;
|
||||
// Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
|
||||
// the window end — the window may hold more, and "all values" is never answered from a truncated read.
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
|
||||
"or narrow the window.")
|
||||
{ PreWindowRows = Crowding(all.Result.PreWindowRows) };
|
||||
}
|
||||
|
||||
var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
|
||||
var pass = await PassAsync(asked).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
|
||||
// A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again. The
|
||||
// refusal carries the pre-window rows it saw (#694): the server re-reads the page that many values smaller.
|
||||
if (pass.Budget < asked)
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
|
||||
$"({pass.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
|
||||
"NumValuesPerNode or narrow the window.")
|
||||
{ PreWindowRows = Crowding(pass.Result.PreWindowRows) };
|
||||
|
||||
// The margin crowded the window out: re-read with room for the pre-window rows. A read that was itself cut short saw
|
||||
// only some of them (#694 F4: a tie cluster of hundreds just before a resume point used to fail even with no
|
||||
// ceiling), so the room at least doubles per re-read — bounded by MaxCrowdedReReads, and never past a known ceiling.
|
||||
// #694 R2: the room is capped at the tie-cluster limit (MaxTieClusterOverfetch) — a pathological single-millisecond
|
||||
// cluster on a streaming gateway would otherwise be streamed several times over for one page.
|
||||
var preWindow = pass.Result.PreWindowRows;
|
||||
var room = Math.Min(Math.Max(1, preWindow), _maxPreWindowRows);
|
||||
for (var attempt = 0; attempt < MaxCrowdedReReads && preWindow <= _maxPreWindowRows; attempt++)
|
||||
{
|
||||
var budget = (int)Math.Min(int.MaxValue, (long)cap + room + RawAggregateReader.PreWindowAllowance);
|
||||
// #695: doubling can leap over the ceiling while the cluster still fits. Spend the remaining room once.
|
||||
if (_ceiling.Learned is var known and > 0 && budget > known)
|
||||
{
|
||||
var available = known - cap - RawAggregateReader.PreWindowAllowance;
|
||||
if (available <= 0) break;
|
||||
room = Math.Min(room, available);
|
||||
budget = cap + room + RawAggregateReader.PreWindowAllowance;
|
||||
}
|
||||
if (budget <= pass.Budget) break; // no more room, never retry the same full read
|
||||
pass = await PassAsync(budget).ConfigureAwait(false);
|
||||
if (pass.Result.Complete) return pass.Result.Samples;
|
||||
preWindow = Math.Max(preWindow, pass.Result.PreWindowRows);
|
||||
if (pass.Budget < budget) break; // bounded by a ceiling learned meanwhile
|
||||
if (room >= _maxPreWindowRows) break; // the limit's worth of room did not suffice
|
||||
room = (int)Math.Min(_maxPreWindowRows, Math.Max(preWindow, 2L * room));
|
||||
}
|
||||
|
||||
throw new HistoryReadLimitExceededException(cap,
|
||||
$"The raw read of '{fullReference}' could not be read completely: more than {pass.Result.PreWindowRows} rows lie in " +
|
||||
$"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.")
|
||||
{ PreWindowRows = preWindow };
|
||||
|
||||
// Pre-window rows the allowance did not cover are what crowded the read; fewer are not the cause (#694).
|
||||
static int Crowding(int preWindowRows) => preWindowRows > RawAggregateReader.PreWindowAllowance ? preWindowRows : 0;
|
||||
|
||||
// One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
|
||||
Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
|
||||
_ceiling.ReadAsync(budget,
|
||||
b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken),
|
||||
(r, _) => !r.Complete);
|
||||
}
|
||||
|
||||
/// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
|
||||
private async Task<(List<HistorianSample> Samples, int PreWindowRows, bool Complete)> ReadRawPassAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, int cap, int budget, CancellationToken cancellationToken)
|
||||
{
|
||||
var samples = new List<HistorianSample>();
|
||||
long received = 0;
|
||||
var preWindow = 0;
|
||||
var reachedEnd = false;
|
||||
await foreach (var sample in _client
|
||||
.ReadRawAsync(fullReference, RawAggregateReader.RequestStartFor(startUtc), endUtc, budget, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
// Filter while draining — the rest of the (bounded) stream is consumed, never abandoned.
|
||||
if (RawAggregateReader.IsWindowSample(sample, startUtc, endUtc))
|
||||
{
|
||||
if (cap == 0 || samples.Count < cap) samples.Add(sample);
|
||||
}
|
||||
// A row before the window — or, for a window starting at/before the epoch (no look-back), AVEVA's synthetic row at
|
||||
// the query start, which is no sample either (#694 F1: it used to set reachedEnd, so a ceiling-truncated read
|
||||
// passed as complete) — is a pre-window row.
|
||||
else if (sample.Timestamp is { } ts
|
||||
&& (ts.ToDateTime() < startUtc || !RawAggregateReader.IsAfterFloor(ts.ToDateTime()))) preWindow++;
|
||||
else if (sample.Timestamp is not null) reachedEnd = true;
|
||||
}
|
||||
|
||||
// cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
|
||||
// gateway's ceiling became the budget — #691.3) or past the window end.
|
||||
var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
|
||||
return (samples, preWindow, complete);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Per-sample aggregates (#681)</b> — Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime — are evaluated from a raw read with the SQL backends' shared evaluator
|
||||
/// (<see cref="RawAggregateReader"/>): identical values, status, Calculated/Partial bits and timestamps to
|
||||
/// SqlServer/QuestDB for the same samples, forward and reverse (#677). The in-window raw rows are capped by
|
||||
/// <c>ServerHistorian:MaxTimeWeightedRawRows</c> (<see cref="HistoryReadLimitExceededException"/>, not counted as
|
||||
/// a backend failure).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>TimeAverage / Total</b> stay native AVEVA retrieval modes (<see cref="AggregateModeMapper"/>), stamped with
|
||||
/// the interval start and the Calculated bit; AVEVA's extra row for the interval before the window is dropped and
|
||||
/// an Uncertain interval keeps its value (#685). Both directions use the SQL backends' interval grid with forward
|
||||
/// requests only — a reverse read (which the gateway refuses) anchored at its later time (#677), the clipped interval
|
||||
/// read over exactly its span and marked Partial (#691.2, <see cref="NativeAggregatePlan"/>). Everything else is <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (see <see cref="ServedAggregates"/>).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <exception cref="ArgumentOutOfRangeException">A raw-evaluated read whose window holds more than
|
||||
/// <see cref="BucketPlan.MaxBuckets"/> intervals.</exception>
|
||||
public async Task<HistoryReadResult> ReadProcessedAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
if (RawAggregateReader.Serves(aggregate))
|
||||
return await ReadProcessedFromRawAsync(fullReference, startUtc, endUtc, interval, aggregate, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count (#285).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
return await ReadNativeAsync(fullReference, BucketPlan.Create(startUtc, endUtc, interval), mode, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A native TimeAverage / Total read on the SQL backends' interval grid, either direction (#691.2): forward native
|
||||
/// requests only, the clipped interval read over exactly its span. See <see cref="NativeAggregatePlan"/>.
|
||||
/// </summary>
|
||||
private async Task<HistoryReadResult> ReadNativeAsync(string fullReference, BucketPlan plan, RetrievalMode mode,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
try
|
||||
{
|
||||
// Total/Count are now native gateway retrieval modes — no client-side scaling
|
||||
// (unlike the Wonderware path that derived Total as Average × interval-seconds).
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, startUtc, endUtc, mode, interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
.ReadAggregateAsync(fullReference, request.StartUtc, request.EndUtc, mode, request.Interval, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
buckets.Add(bucket); // drained to completion, mapped afterwards
|
||||
}
|
||||
NativeAggregatePlan.Place(plan, request, buckets, byBucket);
|
||||
}
|
||||
|
||||
var snapshots = SampleMapper.ToAggregateSnapshots(buckets);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(snapshots, ContinuationPoint: null);
|
||||
return new HistoryReadResult(NativeAggregatePlan.Assemble(plan, byBucket), ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Serves a per-sample aggregate from a raw read (#681); see <see cref="RawAggregateReader"/>.</summary>
|
||||
private async Task<HistoryReadResult> ReadProcessedFromRawAsync(
|
||||
string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
|
||||
HistoryAggregateType aggregate, CancellationToken cancellationToken)
|
||||
{
|
||||
// Planned OUTSIDE the failure-recording try (as on the SQL backends): too many intervals is a client request
|
||||
// limit, not a backend fault.
|
||||
var plan = BucketPlan.Create(startUtc, endUtc, interval);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
|
||||
}
|
||||
var values = await RawAggregateReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
// A client-request limit (the raw-row cap), not a backend fault: not counted as a failure.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (IsMaxValuesCeilingRejection(ex))
|
||||
{
|
||||
// The gateway refused the row budget (Historian:ReadLimits:MaxValuesCeiling with
|
||||
// AllowStreamingBeyondCeiling=false): the same client-request limit as the cap, not a backend fault.
|
||||
throw new HistoryReadLimitExceededException(_maxRawRows,
|
||||
$"The HistorianGateway refused the raw row budget of the processed read of '{fullReference}' (its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling); lower ServerHistorian:MaxTimeWeightedRawRows below that ceiling " +
|
||||
"or narrow the window.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: served from RAW gateway reads — each requested time's stored sample at-or-before and after it — evaluated with
|
||||
/// the SQL backends' shared <see cref="AtTimeEvaluator"/> (Part 11 §6.5.4), so the Gateway answers what SqlServer and
|
||||
/// QuestDB answer for the same stored samples; see <see cref="GatewayAtTimeReader"/> for the read plan and its cost.
|
||||
/// AVEVA's own interpolated answer (the gateway's <c>ReadAtTime</c>) returned the previous value as Good at a stored
|
||||
/// NULL's time, null + Bad at a Good sample right after a NULL, a drifted value at an exact stored sample, and no
|
||||
/// Interpolated bit; it is only used when <c>ServerHistorian:GatewayAtTimeFromRaw = false</c> (kill switch).
|
||||
/// A gateway value is a Double (or a String) whatever the node's type (AVEVA stores a Boolean as a 0/1 <c>Int2</c>), so
|
||||
/// without a type hint every numeric tag is interpolated linearly; the node manager passes the node's type through
|
||||
/// <see cref="ReadAtTimeAsync(string, IReadOnlyList{DateTime}, TypeCode?, CancellationToken)"/>.
|
||||
/// </remarks>
|
||||
public Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken) =>
|
||||
ReadAtTimeAsync(fullReference, timestampsUtc, valueType: null, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// #693: the hint makes the AtTime rules type-exact, as on the SQL backends — the gateway's Double rows are converted to
|
||||
/// the node's type BEFORE the shared evaluator runs (Boolean = value ≠ 0 ⇒ stepped; an integer type ⇒ Int64 ⇒
|
||||
/// interpolated and rounded half away from zero, then returned as the node's integer type; String ⇒ stepped; Float /
|
||||
/// Double / no hint ⇒ linear Double). Ignored with the kill switch (AVEVA's own answer).
|
||||
/// </remarks>
|
||||
public async Task<HistoryReadResult> ReadAtTimeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, TypeCode? valueType, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timestampsUtc);
|
||||
if (!_atTimeFromRaw) return await ReadAtTimeNativeAsync(fullReference, timestampsUtc, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, now);
|
||||
}
|
||||
var values = await GatewayAtTimeReader
|
||||
.ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
|
||||
.ConfigureAwait(false);
|
||||
RecordOutcome(success: true, error: null);
|
||||
return new HistoryReadResult(values, ContinuationPoint: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw; // caller-requested, not a backend fault
|
||||
}
|
||||
catch (HistoryReadLimitExceededException)
|
||||
{
|
||||
throw; // a client-request limit (a pathological tie cluster), not a backend fault
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The pre-#693 AtTime: AVEVA's own interpolation, aligned to the request (kill switch only).</summary>
|
||||
private async Task<HistoryReadResult> ReadAtTimeNativeAsync(
|
||||
string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
RecordOutcome(success: true, error: null);
|
||||
return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
|
||||
}
|
||||
|
||||
var samples = await _client
|
||||
.ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
@@ -121,10 +527,56 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c> (Gitea #686): the gateway's <c>ReadRaw</c> streams ascending only and rejects <c>end < start</c>
|
||||
/// (<c>InvalidArgument</c>). The server serves reverse and end-only capped raw reads from bounded forward reads
|
||||
/// instead, and never sends this source a reverse window (<see cref="ReadRawAsync"/> refuses one unsent).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseRaw => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Gitea #694: <c>0</c> until a strict gateway (<c>AllowStreamingBeyondCeiling=false</c>) has refused a budget and
|
||||
/// named its ceiling (#691.3, <see cref="GatewayRowCeiling"/>); then that ceiling less the
|
||||
/// <see cref="RawAggregateReader.PreWindowAllowance"/> a capped read adds for AVEVA's boundary row — so a page of this
|
||||
/// size asks for exactly the ceiling. Back to <c>0</c> when a re-probe finds the gateway streaming again.
|
||||
/// </remarks>
|
||||
public int MaxRawValuesPerRead => _ceiling.Learned is var c and > 0
|
||||
? Math.Max(1, c - RawAggregateReader.PreWindowAllowance)
|
||||
: 0;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <c>false</c>: the gateway's <c>ReadEvents</c> streams ascending only (Gitea #675). The server never sends it a
|
||||
/// reverse read; since #691.4 it walks one back from bounded forward reads (ties sorted by EventId — the gateway
|
||||
/// orders by event time only).
|
||||
/// </remarks>
|
||||
public bool SupportsReverseEvents => false;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks><see cref="ServedAggregates"/>; every other member answers <c>BadAggregateNotSupported</c> and is not advertised.</remarks>
|
||||
public IReadOnlySet<HistoryAggregateType> SupportedAggregates => ServedAggregates;
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> (Gitea #675) — the gateway's
|
||||
/// <c>ReadEvents</c> streams ascending only and rejects <c>end < start</c>. The server never calls this with
|
||||
/// <c>reverse</c>: it walks a reverse read back from bounded forward reads (#691.4). Nothing is sent.
|
||||
/// </remarks>
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
|
||||
CancellationToken cancellationToken) =>
|
||||
reverse
|
||||
? throw new NotSupportedException(
|
||||
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
@@ -133,7 +585,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
var collected = new List<HistoricalEvent>(hasCap ? maxEvents : 0);
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded", #691.4 review).
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
@@ -170,6 +623,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
catch (Exception ex)
|
||||
{
|
||||
RecordReadFailure(ex);
|
||||
if (IsTransportFailure(ex)) throw new System.IO.IOException("HistorianGateway read transport is unavailable.", ex);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -201,7 +655,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles a gateway at-time reply against the requested timestamps to honour the
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync"/> contract: exactly one snapshot per
|
||||
/// <see cref="IHistorianDataSource.ReadAtTimeAsync(string, IReadOnlyList{DateTime}, CancellationToken)"/> contract: exactly one snapshot per
|
||||
/// requested timestamp, in request order. Returned samples are indexed by timestamp ticks;
|
||||
/// any requested timestamp the gateway did not return is filled with a Bad-quality
|
||||
/// (<c>0x80000000</c>) snapshot stamped at the requested time rather than positionally
|
||||
@@ -244,6 +698,19 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
|
||||
private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
|
||||
new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
|
||||
|
||||
/// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
|
||||
private static bool IsTransportFailure(Exception ex) => ex switch
|
||||
{
|
||||
ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException => true,
|
||||
RpcException rpc => rpc.StatusCode is Grpc.Core.StatusCode.Unavailable or Grpc.Core.StatusCode.DeadlineExceeded,
|
||||
TimeoutException => true,
|
||||
_ => ex.InnerException is { } inner && IsTransportFailure(inner),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Records a failed read: bumps the health counters and logs a generic, redaction-safe
|
||||
/// debug line (no tag, host, or value). The exception itself is rethrown by the caller.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IHistorianBackendProvider"/> for the HistorianGateway sidecar. Numeric-only
|
||||
/// (<c>Double</c>/<c>Int64</c>/<c>Boolean</c>) and read/alarm/provisioning only — no HistoryUpdate.
|
||||
/// </summary>
|
||||
public sealed class GatewayHistorianProvider : IHistorianBackendProvider
|
||||
{
|
||||
private readonly ServerHistorianOptions _options;
|
||||
|
||||
/// <summary>Creates the provider over the bound <c>ServerHistorian</c> options.</summary>
|
||||
/// <param name="options">The gateway connection options.</param>
|
||||
public GatewayHistorianProvider(ServerHistorianOptions options)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string Name => "Gateway";
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// The recorder's own default set (<see cref="ContinuousHistorizationRecorder.DefaultSupportedKinds"/>), so the
|
||||
/// gateway's kinds and the recorder's fallback can never drift apart.
|
||||
/// </remarks>
|
||||
public IReadOnlySet<HistorianValueKind> SupportedKinds => ContinuousHistorizationRecorder.DefaultSupportedKinds;
|
||||
|
||||
/// <inheritdoc />
|
||||
public IHistorianDataSource CreateDataSource(IServiceProvider sp) => GatewayHistorian.CreateDataSource(_options, sp);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IHistorianValueWriter CreateValueWriter(IServiceProvider sp)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sp);
|
||||
var loggerFactory = sp.GetService<ILoggerFactory>() ?? NullLoggerFactory.Instance;
|
||||
var logger = sp.GetService<ILogger<GatewayHistorianValueWriter>>()
|
||||
?? NullLogger<GatewayHistorianValueWriter>.Instance;
|
||||
return new GatewayHistorianValueWriter(HistorianGatewayClientAdapter.Create(_options, loggerFactory), logger);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAlarmHistorianWriter CreateAlarmWriter(IServiceProvider sp) => GatewayHistorian.CreateAlarmWriter(_options, sp);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IHistorianProvisioning CreateProvisioning(IServiceProvider sp) => GatewayHistorian.CreateProvisioner(_options, sp);
|
||||
|
||||
/// <inheritdoc />
|
||||
public IHistorianDataUpdater? CreateUpdater(IServiceProvider sp) => null;
|
||||
}
|
||||
+7
-1
@@ -38,7 +38,13 @@ public static class GatewayHistorian
|
||||
|
||||
return new GatewayHistorianDataSource(
|
||||
HistorianGatewayClientAdapter.Create(options, loggerFactory),
|
||||
logger);
|
||||
logger,
|
||||
// #681: the raw-evaluated per-sample aggregates share the SQL backends' raw-row cap.
|
||||
maxRawRows: options.MaxTimeWeightedRawRows,
|
||||
// #693: AtTime from raw reads + the shared evaluator; false = AVEVA's own interpolation (kill switch).
|
||||
atTimeFromRaw: options.GatewayAtTimeFromRaw,
|
||||
// #694 R2: a crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
|
||||
maxPreWindowRows: options.MaxTieClusterOverfetch);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
|
||||
/// from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
|
||||
/// any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
|
||||
/// — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
|
||||
/// no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget, and the ceiling is
|
||||
/// remembered per data source (<see cref="GatewayRowCeiling"/>); the caller treats a stream that delivered the whole
|
||||
/// ceiling as possibly truncated (never answered from).
|
||||
/// </remarks>
|
||||
internal static class GatewayReadLimits
|
||||
{
|
||||
private const string CeilingRejectionText = "max_values exceeds the server ceiling";
|
||||
|
||||
private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
|
||||
RegexOptions.CultureInvariant | RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
|
||||
/// is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <returns>Whether it is the ceiling rejection.</returns>
|
||||
public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
|
||||
|
||||
/// <summary>
|
||||
/// The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
|
||||
/// carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
|
||||
/// </summary>
|
||||
/// <param name="ex">The exception.</param>
|
||||
/// <param name="ceiling">The ceiling (> 0) when this returns true.</param>
|
||||
/// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
|
||||
public static bool TryGetCeiling(Exception ex, out int ceiling)
|
||||
{
|
||||
ceiling = 0;
|
||||
if (Rejection(ex) is not { } rpc) return false;
|
||||
var m = CeilingNumber.Match(rpc.Status.Detail);
|
||||
ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
|
||||
? n
|
||||
: RawAggregateReader.GatewayDefaultMaxValuesCeiling;
|
||||
return true;
|
||||
}
|
||||
|
||||
private static RpcException? Rejection(Exception ex)
|
||||
{
|
||||
for (var e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
|
||||
&& rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
|
||||
return rpc;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A strict gateway's row ceiling as learned by ONE data source (#691.3 review): every refused attempt counts as an
|
||||
/// error in the gateway's own outcome metrics, so the ceiling a refusal names is remembered and later reads ask for at
|
||||
/// most it — no refusal per read in the steady state. Thread-safe (one <see cref="int"/>, read/written atomically; a
|
||||
/// race costs at most one extra refusal or re-probe, never a wrong answer).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rules.</b> (1) Unknown ceiling: ask for the full budget; a refusal naming <c>N</c> < budget learns <c>N</c> and
|
||||
/// the read is re-issued once at <c>N</c>. (2) Learned ceiling <c>C</c> below the budget: ask for <c>C</c> — a refusal
|
||||
/// there (the gateway was LOWERED) re-learns its <c>N</c> as in (1). (3) If that <c>C</c>-bounded read came back
|
||||
/// full but not provably complete, the gateway may have been RAISED (or switched to streaming) since <c>C</c> was
|
||||
/// learned, and answering <c>BadTooManyOperations</c> from a stale ceiling would be wrong: the read is re-probed ONCE
|
||||
/// at the full budget — served ⇒ the ceiling is forgotten; refused naming a larger <c>N</c> ⇒ <c>N</c> is learned and
|
||||
/// the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
|
||||
/// aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
|
||||
/// limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
|
||||
{
|
||||
private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
|
||||
private long _lastRefusalTicks;
|
||||
private int _learned; // 0 = unknown / not strict
|
||||
|
||||
/// <summary>The learned ceiling (0 = none).</summary>
|
||||
public int Learned => Volatile.Read(ref _learned);
|
||||
|
||||
/// <summary>Runs a drained read under the rules above.</summary>
|
||||
/// <typeparam name="T">The pass result.</typeparam>
|
||||
/// <param name="budget">The <c>max_values</c> the read wants.</param>
|
||||
/// <param name="pass">One drained read with the given budget.</param>
|
||||
/// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
|
||||
/// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
|
||||
/// <returns>The result and the budget it was read with.</returns>
|
||||
public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
|
||||
{
|
||||
var learned = Learned;
|
||||
if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
|
||||
|
||||
var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
|
||||
if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
|
||||
|
||||
// #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
|
||||
if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
|
||||
return bounded;
|
||||
|
||||
// Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
|
||||
int fresh;
|
||||
try
|
||||
{
|
||||
var full = await pass(budget).ConfigureAwait(false);
|
||||
Interlocked.CompareExchange(ref _learned, 0, learned);
|
||||
return (full, budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out fresh) && fresh < budget)
|
||||
{
|
||||
// fall through (outside the catch)
|
||||
}
|
||||
Learn(fresh);
|
||||
return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
|
||||
}
|
||||
|
||||
private void Learn(int ceiling)
|
||||
{
|
||||
Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
|
||||
Volatile.Write(ref _learned, ceiling);
|
||||
}
|
||||
|
||||
private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
|
||||
{
|
||||
int ceiling;
|
||||
try
|
||||
{
|
||||
return (await pass(budget).ConfigureAwait(false), budget);
|
||||
}
|
||||
catch (Exception ex) when (GatewayReadLimits.TryGetCeiling(ex, out ceiling) && ceiling < budget)
|
||||
{
|
||||
// fall through to the retry (outside the catch, so a second refusal is not nested in this one)
|
||||
}
|
||||
Learn(ceiling);
|
||||
return (await pass(ceiling).ConfigureAwait(false), ceiling);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
using Grpc.Core;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
|
||||
internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
|
||||
{
|
||||
internal const int Capacity = 4096;
|
||||
private readonly object _sync = new();
|
||||
private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
|
||||
|
||||
public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
|
||||
|
||||
bool exists;
|
||||
try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
|
||||
catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lock (_sync)
|
||||
{
|
||||
if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
|
||||
_entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
|
||||
}
|
||||
return exists;
|
||||
}
|
||||
|
||||
private static bool IsNotFound(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,14 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
/// </summary>
|
||||
private const uint DefaultStorageRateMs = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Integral divisor stamped on every provisioned analog tag (#677). AVEVA's <c>Total</c> integral is
|
||||
/// <c>∫ value dt (seconds) ÷ IntegralDivisor</c>; the proto's <c>double</c> default of 0 would reach the gateway as
|
||||
/// "unset" (or a divide-by-zero), so 1 is sent explicitly — the value·seconds unit the SQL backends' <c>Total</c>
|
||||
/// uses. (The live magnitude check against AVEVA is still open: it needs the VPN.)
|
||||
/// </summary>
|
||||
private const double DefaultIntegralDivisor = 1.0;
|
||||
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayTagProvisioner> _logger;
|
||||
|
||||
@@ -70,6 +78,8 @@ public sealed class GatewayTagProvisioner : IHistorianProvisioning, IAsyncDispos
|
||||
Description = request.Description ?? string.Empty,
|
||||
// Required: a 0 storage rate makes the gateway's EnsureTags throw (see the constant).
|
||||
StorageRateMs = DefaultStorageRateMs,
|
||||
// Explicit: the proto3 double default (0) is not "1" (see the constant).
|
||||
IntegralDivisor = DefaultIntegralDivisor,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+11
@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
|
||||
{
|
||||
// #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
|
||||
// absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
|
||||
var found = false;
|
||||
await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
|
||||
found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
|
||||
return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncEnumerable<HistorianSample> ReadRawAsync(
|
||||
string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>
|
||||
|
||||
@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// </summary>
|
||||
public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
{
|
||||
/// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>Whether the historian contains the tag.</returns>
|
||||
Task<bool> TagExistsAsync(string tag, CancellationToken ct);
|
||||
|
||||
/// <summary>Streams raw historian samples for a tag over a time window.</summary>
|
||||
/// <param name="tag">The historian tag name to read.</param>
|
||||
/// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>
|
||||
|
||||
+35
-17
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Frozen;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
@@ -5,40 +6,57 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
/// <summary>
|
||||
/// Maps the driver-agnostic <see cref="HistoryAggregateType"/> (OPC UA Part 13 aggregate) onto the
|
||||
/// gateway's native <see cref="RetrievalMode"/>.
|
||||
/// gateway's native <see cref="RetrievalMode"/> — for the aggregates still answered by AVEVA itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Average/Minimum/Maximum line up with the legacy Wonderware client's aggregate mapping. The two
|
||||
/// remaining members are now served by <b>native</b> gateway retrieval modes:
|
||||
/// <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/> and
|
||||
/// <see cref="HistoryAggregateType.Count"/> → <see cref="RetrievalMode.Counter"/>.
|
||||
/// Native (AVEVA-computed): <see cref="HistoryAggregateType.TimeAverage"/> → <see cref="RetrievalMode.TimeWeightedAverage"/>
|
||||
/// and <see cref="HistoryAggregateType.Total"/> → <see cref="RetrievalMode.Integral"/>. They stay native because an exact
|
||||
/// Part 13 time-weighted evaluation needs the real bounding samples on both sides of the window (time, value and
|
||||
/// quality of the latest sample before it), and the gateway has no "previous stored sample" read — a raw read streams
|
||||
/// forward from its start and AVEVA replaces whatever lies before with an initial-value row stamped at the query start,
|
||||
/// losing the original time a sloped segment across the window start needs (#681).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This replaces the Wonderware-era client-side workarounds (Total derived as Average × interval,
|
||||
/// Count approximated from a value count): no client-side scaling is performed any more, so the
|
||||
/// gateway path is a strict improvement.
|
||||
/// Since #681 the per-sample aggregates (Average, Count, Minimum, Maximum, Range, Start, End, Delta,
|
||||
/// Minimum/MaximumActualTime) are NOT native: <see cref="GatewayHistorianDataSource"/> evaluates them from raw reads
|
||||
/// with the SQL backends' shared evaluator (Minimum/Maximum used to map to AVEVA's <c>MinimumWithTime</c> /
|
||||
/// <c>MaximumWithTime</c>). Every other member — incl. TimeAverage2, Total2, Interpolative, PercentGood, PercentBad — is
|
||||
/// <see cref="HistoryAggregateNotSupportedException"/> here.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class AggregateModeMapper
|
||||
{
|
||||
/// <summary>
|
||||
/// The aggregates answered by a native AVEVA retrieval mode — every defined member <see cref="TryMap"/> maps, derived
|
||||
/// from it so the dispatch and the declared set can never disagree. Part of
|
||||
/// <see cref="GatewayHistorianDataSource.ServedAggregates"/> (#681/#682).
|
||||
/// </summary>
|
||||
public static readonly FrozenSet<HistoryAggregateType> NativeAggregates =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(a => TryMap(a) is not null).ToFrozenSet();
|
||||
|
||||
/// <summary>Maps an aggregate function to the gateway retrieval mode.</summary>
|
||||
/// <param name="aggregate">The driver-agnostic aggregate function.</param>
|
||||
/// <returns>The matching gateway <see cref="RetrievalMode"/>.</returns>
|
||||
/// <exception cref="HistoryAggregateNotSupportedException">A defined member the gateway has no retrieval mode for.</exception>
|
||||
/// <exception cref="ArgumentOutOfRangeException">An undefined enum value.</exception>
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate) => aggregate switch
|
||||
public static RetrievalMode ToRetrievalMode(HistoryAggregateType aggregate)
|
||||
{
|
||||
HistoryAggregateType.Average => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Minimum => RetrievalMode.MinimumWithTime,
|
||||
HistoryAggregateType.Maximum => RetrievalMode.MaximumWithTime,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
HistoryAggregateType.Count => RetrievalMode.Counter,
|
||||
if (TryMap(aggregate) is { } mode) return mode;
|
||||
|
||||
// Part 13 aggregates the gateway has no retrieval mode for (#285) — an UNDEFINED enum value is a
|
||||
// programming error, but a defined-yet-unserved member is a normal client request the server
|
||||
// answers with BadAggregateNotSupported.
|
||||
_ when Enum.IsDefined(aggregate) => throw new HistoryAggregateNotSupportedException(aggregate),
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping."),
|
||||
if (Enum.IsDefined(aggregate)) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(aggregate), aggregate, "Unmapped HistoryAggregateType — add a RetrievalMode mapping.");
|
||||
}
|
||||
|
||||
/// <summary>The single mapping table: the retrieval mode for a natively served aggregate, or null.</summary>
|
||||
private static RetrievalMode? TryMap(HistoryAggregateType aggregate) => aggregate switch
|
||||
{
|
||||
HistoryAggregateType.TimeAverage => RetrievalMode.TimeWeightedAverage,
|
||||
HistoryAggregateType.Total => RetrievalMode.Integral,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,16 +35,16 @@ internal static class AlarmEventMapper
|
||||
/// <returns>The gateway wire event ready for <c>SendEvent</c>.</returns>
|
||||
public static HistorianEvent ToHistorianEvent(AlarmHistorianEvent alarm)
|
||||
{
|
||||
if (alarm.DeliveryId == Guid.Empty)
|
||||
throw new ArgumentException("Alarm delivery identity must be nonzero.", nameof(alarm));
|
||||
// Timestamp.FromDateTime requires a Utc-kind DateTime; coerce defensively (TimestampUtc is
|
||||
// already Utc by contract, but a caller could pass Unspecified).
|
||||
var eventTime = Timestamp.FromDateTime(DateTime.SpecifyKind(alarm.TimestampUtc, DateTimeKind.Utc));
|
||||
|
||||
var historianEvent = new HistorianEvent
|
||||
{
|
||||
// Deliberately DO NOT set HistorianEvent.Id: the gateway's SendEvent path rejects a
|
||||
// client-supplied event id (the server handler throws and the call fails permanently —
|
||||
// confirmed live). The historian assigns event identity server-side; the alarm's own id
|
||||
// is preserved below as a property for read-back correlation/traceability.
|
||||
// Identity belongs to the event/durable queue, never to an individual RPC attempt.
|
||||
Id = alarm.DeliveryId.ToString("D"),
|
||||
SourceName = alarm.EquipmentPath,
|
||||
Type = alarm.AlarmTypeName,
|
||||
EventTime = eventTime,
|
||||
|
||||
+32
@@ -43,4 +43,36 @@ internal static class GatewayQualityMapper
|
||||
_ when q >= 64 => HistorianStatusCodes.Uncertain,
|
||||
_ => HistorianStatusCodes.Bad,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Reverse of <see cref="Map"/>: maps an OPC UA StatusCode to the OPC DA quality byte the gateway
|
||||
/// expects. Every explicit <see cref="Map"/> entry round-trips; an unknown status falls back to its
|
||||
/// severity bucket (Good 192, Uncertain 64, Bad 0).
|
||||
/// </summary>
|
||||
/// <param name="statusCode">The OPC UA StatusCode.</param>
|
||||
/// <returns>The OPC DA quality byte.</returns>
|
||||
public static byte ToDaQuality(uint statusCode) => statusCode switch
|
||||
{
|
||||
HistorianStatusCodes.Good => 192,
|
||||
HistorianStatusCodes.GoodLocalOverride => 216,
|
||||
HistorianStatusCodes.Uncertain => 64,
|
||||
HistorianStatusCodes.UncertainLastUsableValue => 68,
|
||||
HistorianStatusCodes.UncertainSensorNotAccurate => 80,
|
||||
HistorianStatusCodes.UncertainEngineeringUnitsExceeded => 84,
|
||||
HistorianStatusCodes.UncertainSubNormal => 88,
|
||||
HistorianStatusCodes.Bad => 0,
|
||||
HistorianStatusCodes.BadConfigurationError => 4,
|
||||
HistorianStatusCodes.BadNotConnected => 8,
|
||||
HistorianStatusCodes.BadDeviceFailure => 12,
|
||||
HistorianStatusCodes.BadSensorFailure => 16,
|
||||
HistorianStatusCodes.BadCommunicationError => 20,
|
||||
HistorianStatusCodes.BadOutOfService => 24,
|
||||
HistorianStatusCodes.BadWaitingForInitialData => 32,
|
||||
_ => (statusCode >> 30) switch
|
||||
{
|
||||
0 => (byte)192,
|
||||
1 => (byte)64,
|
||||
_ => (byte)0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
|
||||
@@ -16,30 +17,50 @@ internal static class SampleMapper
|
||||
/// <summary>OPC DA "Good" family floor — a quality byte at/above this carries usable data.</summary>
|
||||
private const byte GoodQualityFloor = 192;
|
||||
|
||||
/// <summary>OPC DA "Uncertain" family floor (64–191).</summary>
|
||||
private const byte UncertainQualityFloor = 64;
|
||||
|
||||
/// <summary>Maps a single raw sample to a value snapshot.</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// #690: a Bad row carries no value. histsdk puts a value on EVERY row (<c>NumericValue = 0</c> on a numeric row,
|
||||
/// <c>StringValue = ""</c> on a string one), so a stored NULL arrives as <c>0</c> / <c>""</c> with a Bad OPC quality
|
||||
/// (live: <c>q=1 opc=0</c>) — that 0 is not a value, and AVEVA stores a Bad-quality sample as NULL, so a Bad row never
|
||||
/// carries a real one. The value is therefore dropped whenever the mapped status is Bad, matching the SqlServer/QuestDB
|
||||
/// backends (an Empty sample is null + Bad) and the aggregate mapper's no-data rows. Uncertain rows keep their value.
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToSnapshot(HistorianSample sample)
|
||||
{
|
||||
var status = ToStatus(sample);
|
||||
// proto3 explicit presence: prefer the numeric value, else the string value, else null.
|
||||
object? value;
|
||||
if (sample.HasNumericValue)
|
||||
if (StatusBits.IsBad(status))
|
||||
value = null;
|
||||
else if (sample.HasNumericValue)
|
||||
value = sample.NumericValue; // boxes as System.Double
|
||||
else if (sample.HasStringValue)
|
||||
value = sample.StringValue;
|
||||
else
|
||||
value = null;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: status,
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>The OPC UA status of a raw sample (shared by raw reads and the #681 raw-evaluated aggregates).</summary>
|
||||
/// <param name="sample">The gateway raw sample.</param>
|
||||
/// <returns>The mapped status code.</returns>
|
||||
public static uint ToStatus(HistorianSample sample)
|
||||
{
|
||||
// Prefer the OPC DA quality byte (opc_quality); the gateway populates it directly from the
|
||||
// SDK's OpcQuality, so it is the authoritative byte for GatewayQualityMapper. Fall back to
|
||||
// the historian quality field only when opc_quality is unset (0).
|
||||
byte qualityByte = sample.OpcQuality != 0 ? (byte)sample.OpcQuality : (byte)sample.Quality;
|
||||
|
||||
return new DataValueSnapshot(
|
||||
Value: value,
|
||||
StatusCode: GatewayQualityMapper.Map(qualityByte),
|
||||
SourceTimestampUtc: sample.Timestamp?.ToDateTime(), // Utc kind
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
return GatewayQualityMapper.Map(qualityByte);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of raw samples to value snapshots, in order.</summary>
|
||||
@@ -57,31 +78,78 @@ internal static class SampleMapper
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <returns>The driver-agnostic snapshot.</returns>
|
||||
/// <remarks>
|
||||
/// Unlike the legacy Wonderware DTO (a nullable <c>double?</c>), the gateway proto carries a
|
||||
/// non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled by a
|
||||
/// null value. Instead it is signalled by a non-Good <c>opc_quality</c>: a Good bucket
|
||||
/// (<c>opc_quality >= 192</c>) yields its value with a Good status, anything else maps to
|
||||
/// <c>BadNoData</c> with a null value — preserving the Wonderware aggregate contract (binary
|
||||
/// Good-with-value / BadNoData-null).
|
||||
/// <para>
|
||||
/// The gateway proto carries a non-optional <c>double value</c>, so an unavailable (no-data) bucket cannot be signalled
|
||||
/// by a null value; it is signalled by the OPC DA quality family of <c>opc_quality</c> (its low byte, as the raw mapping
|
||||
/// reads it):
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>Good (≥ 192) ⇒ the value, <c>Good</c>.</item>
|
||||
/// <item>Uncertain (64–191) ⇒ the value, <c>UncertainDataSubNormal</c> (#685): AVEVA marks an interval that holds a
|
||||
/// NULL/Bad sample <c>opc_quality = 64</c> and still answers its value (computed over the interval's good time only —
|
||||
/// see docs/Historian.md). It used to map to <c>BadNoData</c> with the value discarded. The sub-code is the one the SQL
|
||||
/// backends' aggregates give an interval that includes non-good data.</item>
|
||||
/// <item>Bad (< 64) ⇒ <c>BadNoData</c>, null value (the wire's 0 is not a value).</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// #681: every status carries the Calculated info bits (<c>0x401</c>) like every other backend's processed value, and
|
||||
/// the bucket is stamped with its interval START (Part 13; it was the interval end), falling back to the end only when
|
||||
/// the start is absent (verified on live AVEVA 2026-10-12: <c>start_time</c> is the interval start).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static DataValueSnapshot ToAggregateSnapshot(HistorianAggregateSample aggregate)
|
||||
{
|
||||
bool hasData = aggregate.OpcQuality >= GoodQualityFloor;
|
||||
var q = (byte)aggregate.OpcQuality;
|
||||
var status = q >= GoodQualityFloor ? StatusGood
|
||||
: q >= UncertainQualityFloor ? Sql.Common.HistorianStatusCodes.UncertainDataSubNormal
|
||||
: StatusBadNoData;
|
||||
return new DataValueSnapshot(
|
||||
Value: hasData ? aggregate.Value : null, // boxes as System.Double when present
|
||||
StatusCode: hasData ? StatusGood : StatusBadNoData,
|
||||
SourceTimestampUtc: (aggregate.EndTime ?? aggregate.StartTime)?.ToDateTime(), // bucket timestamp
|
||||
Value: status == StatusBadNoData ? null : aggregate.Value, // boxes as System.Double when present
|
||||
StatusCode: HistorianInfoBits.WithCalculated(status),
|
||||
SourceTimestampUtc: (aggregate.StartTime ?? aggregate.EndTime)?.ToDateTime(), // bucket timestamp = interval start
|
||||
ServerTimestampUtc: DateTime.UtcNow);
|
||||
}
|
||||
|
||||
/// <summary>Maps a batch of aggregate buckets to value snapshots, in order.</summary>
|
||||
/// <summary>
|
||||
/// True when the aggregate row's interval lies in the requested window — the span between <paramref name="startUtc"/>
|
||||
/// and <paramref name="endUtc"/> in either order (a reverse request covers the same span). AVEVA answers one extra row
|
||||
/// for the interval BEFORE the window (<c>[start − interval, start)</c>, #685), which Part 11/13 has no place for: an
|
||||
/// interval belongs to the window when its start lies in <c>[lo, hi)</c>; a row without a start time (the mapper then
|
||||
/// stamps it with its end) belongs when its end lies in <c>(lo, hi]</c>.
|
||||
/// </summary>
|
||||
/// <param name="aggregate">The gateway aggregate sample.</param>
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>Whether the row is one of the window's intervals.</returns>
|
||||
public static bool IsInWindow(HistorianAggregateSample aggregate, DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var lo = startUtc <= endUtc ? startUtc : endUtc;
|
||||
var hi = startUtc <= endUtc ? endUtc : startUtc;
|
||||
if (aggregate.StartTime is { } s)
|
||||
{
|
||||
var t = s.ToDateTime();
|
||||
return t >= lo && t < hi;
|
||||
}
|
||||
if (aggregate.EndTime is { } e)
|
||||
{
|
||||
var t = e.ToDateTime();
|
||||
return t > lo && t <= hi;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Maps the aggregate buckets of a window to value snapshots, in order, dropping rows outside it (#685).</summary>
|
||||
/// <param name="aggregates">The gateway aggregate samples.</param>
|
||||
/// <returns>The driver-agnostic snapshots.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates)
|
||||
/// <param name="startUtc">The request's start time.</param>
|
||||
/// <param name="endUtc">The request's end time.</param>
|
||||
/// <returns>The driver-agnostic snapshots, one per in-window interval row.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> ToAggregateSnapshots(IEnumerable<HistorianAggregateSample> aggregates,
|
||||
DateTime startUtc, DateTime endUtc)
|
||||
{
|
||||
var result = new List<DataValueSnapshot>();
|
||||
foreach (var aggregate in aggregates)
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
if (IsInWindow(aggregate, startUtc, endUtc))
|
||||
result.Add(ToAggregateSnapshot(aggregate));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Serves a processed read of the AVEVA-native <c>TimeAverage</c> / <c>Total</c> on the SQL backends' interval grid
|
||||
/// (<see cref="BucketPlan"/>), in either direction: forward — window <c>[start, end)</c>, intervals stepping forward
|
||||
/// from <c>start</c>, the clipped interval the LATEST, each stamped with its earlier edge, oldest first; reverse (#677,
|
||||
/// #691.2) — window <c>(end, start]</c>, intervals stepping back from <c>start</c>, the clipped interval the EARLIEST,
|
||||
/// each stamped with its later edge, newest first. The gateway refuses <c>start > end</c> and <c>interval <= 0</c>,
|
||||
/// so every request it sees is forward with a positive interval.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward requests on the plan's grid.</b> AVEVA anchors a native read's interval grid at the request start (live,
|
||||
/// 2026-10-12: rows at <c>start + j·interval</c>, plus one extra row for the interval before it, #685). So the plan's
|
||||
/// whole intervals are requested as ONE forward read spanning exactly them with the interval <c>I</c>, and the
|
||||
/// clipped interval (when the window is not a whole number of intervals) as a second forward read over exactly its
|
||||
/// span with its own width as the interval — AVEVA never averages time outside the window (before it on a reverse
|
||||
/// read, past <c>EndTime</c> on a forward one). An interval wider than the window is sent as the window, and
|
||||
/// <c>interval <= 0</c> as one interval over the whole window (it used to reach the gateway as 0 and be refused).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Soundness of reverse.</b> A time-weighted average / integral over <c>[a, b)</c> and over <c>(a, b]</c> integrate
|
||||
/// the same function over sets differing by two instants, so a forward native row IS the value of the reverse
|
||||
/// interval with the same edges. The one place the half-open side could matter is AVEVA's status for an interval
|
||||
/// whose only non-good sample lies exactly on an edge — accepted, as on every native row.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mapping.</b> Each row is mapped as before (<see cref="SampleMapper.ToAggregateSnapshot"/>: Uncertain keeps its
|
||||
/// value, no data ⇒ <c>BadNoData</c>, Calculated bits), placed by its interval start (else end), restamped with
|
||||
/// <see cref="BucketPlan.BucketTimestamp"/>, and the clipped interval carries the Partial bit unless Bad (Part 13
|
||||
/// §5.3.3.2 Case 3, as the SQL backends' time-weighted aggregates — which additionally mark the intervals the data
|
||||
/// starts/ends inside; AVEVA's rows carry no data range, so those are not reproduced). An interval AVEVA answers no
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
/// <param name="Interval">The request's interval (> 0).</param>
|
||||
/// <param name="FirstBucket">Ascending plan bucket of the request's first interval.</param>
|
||||
/// <param name="BucketCount">Number of plan buckets the request covers.</param>
|
||||
internal readonly record struct Request(DateTime StartUtc, DateTime EndUtc, TimeSpan Interval, long FirstBucket, long BucketCount);
|
||||
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
var clipLo = new DateTime(plan.LowerTicks(clipped), DateTimeKind.Utc);
|
||||
var clipHi = new DateTime(plan.UpperTicks(clipped), DateTimeKind.Utc);
|
||||
var requests = new List<Request>(2);
|
||||
if (n > 1)
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row, #685)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
/// </summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="request">The forward request the rows answer.</param>
|
||||
/// <param name="rows">The drained rows.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index (first row per bucket wins).</param>
|
||||
public static void Place(BucketPlan plan, Request request, IEnumerable<HistorianAggregateSample> rows,
|
||||
Dictionary<long, DataValueSnapshot> byBucket)
|
||||
{
|
||||
var interval = request.Interval.Ticks;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!SampleMapper.IsInWindow(row, request.StartUtc, request.EndUtc)) continue;
|
||||
long offset = row.StartTime is { } s
|
||||
? (s.ToDateTime().Ticks - request.StartUtc.Ticks) / interval
|
||||
: (row.EndTime.ToDateTime().Ticks - request.StartUtc.Ticks - 1) / interval; // IsInWindow ⇒ end in (lo, hi]
|
||||
if (offset < 0 || offset >= request.BucketCount) continue;
|
||||
var index = request.FirstBucket + offset;
|
||||
if (byBucket.ContainsKey(index)) continue;
|
||||
|
||||
var snapshot = SampleMapper.ToAggregateSnapshot(row);
|
||||
var status = snapshot.StatusCode;
|
||||
if (plan.IsTruncated(index) && !StatusBits.IsBad(status)) status |= HistorianInfoBits.HistorianPartial;
|
||||
byBucket[index] = snapshot with { StatusCode = status, SourceTimestampUtc = plan.BucketTimestamp(index) };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The placed results in the read's order (ascending forward, descending reverse).</summary>
|
||||
/// <param name="plan">The plan.</param>
|
||||
/// <param name="byBucket">Results by ascending bucket index.</param>
|
||||
/// <returns>The results.</returns>
|
||||
public static IReadOnlyList<DataValueSnapshot> Assemble(BucketPlan plan, IReadOnlyDictionary<long, DataValueSnapshot> byBucket) =>
|
||||
(plan.Reversed ? byBucket.OrderByDescending(kv => kv.Key) : byBucket.OrderBy(kv => kv.Key)).Select(kv => kv.Value).ToList();
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// #681: serves the per-sample processed aggregates (the <see cref="AggregateFamily.Simple"/>,
|
||||
/// <see cref="AggregateFamily.StartEnd"/> and <see cref="AggregateFamily.ActualTime"/> families) on the Gateway backend by
|
||||
/// streaming the window's RAW samples through the gateway and evaluating them with the shared
|
||||
/// <c>Driver.Historian.Sql.Common</c> code — <see cref="BucketPlan"/> for the bucket math (forward and reverse, #677) and
|
||||
/// <see cref="AggregateEvaluator.EvaluateAll"/> for values, status and the Partial/Calculated bits — so the Gateway answers
|
||||
/// exactly what SqlServer and QuestDB answer for the same stored samples.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Forward raw reads, each consumed to completion.</b> The gateway's <c>ReadRaw</c> streams oldest-first over an
|
||||
/// inclusive <c>[start, end]</c> and refuses <c>end < start</c>, so a reverse read streams the same ascending range
|
||||
/// and <see cref="BucketPlan"/> assembles the result newest-first. The window stream is classified sample by sample
|
||||
/// with <see cref="BucketPlan.IndexOf"/> (<c>[start, end)</c> forward, <c>(start, end]</c> reverse) — memory is
|
||||
/// O(buckets), never O(samples). No gateway stream is ever abandoned mid-way: a client-side break cancels the call,
|
||||
/// and the gateway currently classifies that cancellation as a connection fault and evicts its shared historian
|
||||
/// session (HistorianGateway <c>HistorianRpcFaults.IsConnectionFault</c>: any <c>RpcException</c> ⇒ true).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The AVEVA start-boundary row.</b> An AVEVA <c>Full</c> raw read emits an initial-value row stamped exactly at
|
||||
/// the query start (histsdk reverse-engineering: <c>Quality = 133</c>, <c>OpcQuality = 192</c>) carrying the value
|
||||
/// in effect there, not a stored sample. The window stream is therefore requested from <see cref="BoundaryMargin"/>
|
||||
/// (1 ms) BEFORE the window, so that row lands outside every bucket and can never be counted as a sample. A row
|
||||
/// before the window counts only as evidence that the tag has earlier data, and only when its OPC quality is not
|
||||
/// Bad — a "no earlier data" NULL row still arrives WITH a value (histsdk sets <c>NumericValue = 0</c> on every
|
||||
/// numeric row, <c>StringValue = ""</c> on a string one) but with a Bad OPC quality (captured: 1, no-data).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The data range.</b> The start/end-of-data Partial rules and Count's in-range empty interval need the tag's
|
||||
/// earliest sample at or before the window's later edge and latest at or after its earlier edge
|
||||
/// (<see cref="AggregateEvaluator.NeedsDataRange"/>; a plain <c>Average</c> needs neither). Earlier data is a non-Bad
|
||||
/// pre-window row, or — when none arrived — a <see cref="ProbeRows"/>-row read <c>[</c><see cref="ProbeFloor"/><c>,
|
||||
/// start)</c>. Later data is a sample exactly at the window end (forward) from the window read, or else a
|
||||
/// <see cref="ProbeRows"/>-row read <c>[end, now + </c><see cref="LookAheadHorizon"/><c>]</c> (the historian stores
|
||||
/// nothing further in the future). In both small reads a row stamped exactly at the query start (a boundary row) or
|
||||
/// at the query end (a possible terminal row) is not a stored sample, whatever its value.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Bounded.</b> The in-window rows are capped by <c>ServerHistorian:MaxTimeWeightedRawRows</c> (the same knob the
|
||||
/// SQL backends' time-weighted reads use): past it the node fails with
|
||||
/// <see cref="HistoryReadLimitExceededException"/> (⇒ <c>BadTooManyOperations</c>) once the stream — bounded by the
|
||||
/// requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
|
||||
/// <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
|
||||
/// or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
|
||||
/// complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
|
||||
/// below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
|
||||
/// then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
|
||||
/// "delivered the whole budget ⇒ cannot be proven complete" rule applies.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
|
||||
/// status is <see cref="SampleMapper.ToStatus"/> — the same quality mapping a raw HistoryRead serves (C-002: the
|
||||
/// gateway stores no quality, so AVEVA history is normally all Good). An Empty sample is clamped Bad
|
||||
/// (<see cref="StoredSample.FromStored"/>), exactly as on the SQL backends.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class RawAggregateReader
|
||||
{
|
||||
/// <summary>How far before the window the raw stream starts, so the AVEVA initial-value row lands outside it.</summary>
|
||||
internal static readonly TimeSpan BoundaryMargin = TimeSpan.FromMilliseconds(1);
|
||||
|
||||
/// <summary>How far past "now" the stream may look for a sample after the window (end-of-data rules).</summary>
|
||||
internal static readonly TimeSpan LookAheadHorizon = TimeSpan.FromDays(1);
|
||||
|
||||
/// <summary>Rows the stream may spend before the window (the boundary row plus samples inside the margin).</summary>
|
||||
internal const int PreWindowAllowance = 16;
|
||||
|
||||
/// <summary>Start of the "is there data before the window?" probe (the Unix epoch — before any AVEVA history).</summary>
|
||||
internal static readonly DateTime ProbeFloor = DateTime.UnixEpoch;
|
||||
|
||||
/// <summary>Rows the earlier-data probe asks for (a possible boundary row plus the first stored sample).</summary>
|
||||
internal const int ProbeRows = 2;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ReadRaw</c> start for a window that begins at <paramref name="windowStartUtc"/>: <see cref="BoundaryMargin"/>
|
||||
/// earlier, so the AVEVA start-boundary row — stamped exactly at the QUERY start, carrying the value in effect there
|
||||
/// (<c>Quality = 133</c>) or, with no earlier data, a NULL row (value 0, Bad OPC quality) — lands BEFORE the window
|
||||
/// (<see cref="IsWindowSample"/>). The one place both the raw-evaluated aggregates (#681) and a raw HistoryRead
|
||||
/// (#687) decide what the request start is. Never before <see cref="ProbeFloor"/> (the Unix epoch — the start of
|
||||
/// the historian's time encoding, the same floor the server's reverse walk-back uses, #686): a window starting less
|
||||
/// than one margin after it is requested from the epoch itself.
|
||||
/// </summary>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <returns>The start to request.</returns>
|
||||
public static DateTime RequestStartFor(DateTime windowStartUtc) =>
|
||||
windowStartUtc - ProbeFloor > BoundaryMargin ? windowStartUtc - BoundaryMargin
|
||||
: windowStartUtc > ProbeFloor ? ProbeFloor
|
||||
: windowStartUtc;
|
||||
|
||||
/// <summary>
|
||||
/// True when a row of a forward raw stream requested from <see cref="RequestStartFor"/> is a stored sample of the raw
|
||||
/// window <c>[windowStartUtc, windowEndUtc)</c> — the <see cref="IHistorianDataSource.ReadRawAsync"/> contract
|
||||
/// (exclusive end, as on the SQL backends; the gateway's own <c>ReadRaw</c> end is inclusive) — or, for a point read
|
||||
/// (<c>start == end</c>, the server's tie-cluster over-fetch), of that one instant. A row before the start is the AVEVA
|
||||
/// boundary row (whatever its value or quality — a stored NULL looks the same on the wire, so the timestamp is the
|
||||
/// evidence) or a stored sample inside the margin; a row without a timestamp is never a sample.
|
||||
/// </summary>
|
||||
/// <param name="raw">The gateway row.</param>
|
||||
/// <param name="windowStartUtc">The window's (inclusive) start.</param>
|
||||
/// <param name="windowEndUtc">The window's end: exclusive, or inclusive when it equals the start.</param>
|
||||
/// <returns>Whether the row belongs to the window.</returns>
|
||||
public static bool IsWindowSample(HistorianSample raw, DateTime windowStartUtc, DateTime windowEndUtc) =>
|
||||
raw.Timestamp is { } ts && ts.ToDateTime() is var t && IsAfterFloor(t) && t >= windowStartUtc
|
||||
&& (t < windowEndUtc || (t == windowEndUtc && windowStartUtc == windowEndUtc));
|
||||
|
||||
/// <summary>
|
||||
/// A row stamped at or before <see cref="ProbeFloor"/> (the Unix epoch) is never a stored sample: AVEVA stores nothing
|
||||
/// there (the server's #686 walk-back assumes the same), and a window starting at or before the epoch gets no 1 ms
|
||||
/// margin (<see cref="RequestStartFor"/>), so AVEVA's synthetic boundary / no-data row lands exactly at its start.
|
||||
/// </summary>
|
||||
/// <param name="t">The row's timestamp.</param>
|
||||
/// <returns>Whether the row can be a stored sample.</returns>
|
||||
internal static bool IsAfterFloor(DateTime t) => t > ProbeFloor;
|
||||
|
||||
/// <summary>True for the aggregates this reader serves: the per-sample families of <see cref="AggregateFamilies"/>.</summary>
|
||||
/// <param name="aggregate">Requested aggregate.</param>
|
||||
/// <returns>Whether the aggregate is evaluated from raw samples.</returns>
|
||||
public static bool Serves(HistoryAggregateType aggregate) =>
|
||||
AggregateFamilies.FamilyOf(aggregate) is AggregateFamily.Simple or AggregateFamily.StartEnd or AggregateFamily.ActualTime;
|
||||
|
||||
/// <summary>The gateway <c>max_values</c> for a configured window cap (<see cref="int.MaxValue"/> when unbounded).</summary>
|
||||
/// <param name="maxWindowRows">The configured cap (<= 0 = unbounded).</param>
|
||||
/// <returns>The row limit to request.</returns>
|
||||
public static int MaxValuesFor(int maxWindowRows) =>
|
||||
maxWindowRows <= 0 ? int.MaxValue : ClampToGatewayCeiling((long)maxWindowRows + 2 + PreWindowAllowance, maxWindowRows);
|
||||
|
||||
/// <summary>
|
||||
/// The HistorianGateway's default <c>Historian:ReadLimits:MaxValuesCeiling</c>. With
|
||||
/// <c>AllowStreamingBeyondCeiling=false</c> the gateway refuses any <c>max_values</c> above its ceiling.
|
||||
/// </summary>
|
||||
internal const int GatewayDefaultMaxValuesCeiling = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// #688: a <c>ReadRaw</c> row budget of <paramref name="rows"/> plus this backend's own allowance
|
||||
/// (<paramref name="requested"/>) clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> — but never below
|
||||
/// <paramref name="rows"/> itself. The allowance (the boundary row, the margin, the completeness row) must not push a
|
||||
/// budget that fits the gateway's default ceiling over it: the default <c>MaxTimeWeightedRawRows</c> (1,000,000) + 18
|
||||
/// used to exceed it, so a gateway with <c>AllowStreamingBeyondCeiling=false</c> refused EVERY per-sample read. A
|
||||
/// count above the ceiling is the operator's (or client's) deliberate choice for a streaming gateway and keeps its
|
||||
/// allowance. The cost of clamping is that a window holding within the allowance of the ceiling cannot be proven
|
||||
/// complete and fails as over the cap — the same outcome it would get one row later.
|
||||
/// </summary>
|
||||
/// <param name="requested">The budget including the allowance.</param>
|
||||
/// <param name="rows">The caller's own row count (the cap, or <c>NumValuesPerNode</c>), > 0.</param>
|
||||
/// <returns>The <c>max_values</c> to request.</returns>
|
||||
internal static int ClampToGatewayCeiling(long requested, long rows) =>
|
||||
(int)Math.Min(int.MaxValue, rows > GatewayDefaultMaxValuesCeiling ? requested : Math.Min(requested, GatewayDefaultMaxValuesCeiling));
|
||||
|
||||
/// <summary>Maps one gateway raw sample to a stored sample (value + mapped status, Empty clamped Bad).</summary>
|
||||
/// <param name="sample">The gateway sample; its timestamp must be set.</param>
|
||||
/// <returns>The stored sample.</returns>
|
||||
public static StoredSample ToStored(HistorianSample sample)
|
||||
{
|
||||
var value = sample.HasNumericValue ? HistorianValue.FromDouble(sample.NumericValue)
|
||||
: sample.HasStringValue ? HistorianValue.FromString(sample.StringValue)
|
||||
: HistorianValue.Empty;
|
||||
return StoredSample.FromStored(sample.Timestamp.ToDateTime(), value, SampleMapper.ToStatus(sample));
|
||||
}
|
||||
|
||||
/// <summary>Reads and evaluates one per-sample processed read.</summary>
|
||||
/// <param name="client">The gateway client.</param>
|
||||
/// <param name="ceiling">The data source's learned gateway row ceiling (#691.3).</param>
|
||||
/// <param name="tag">Historian tag name.</param>
|
||||
/// <param name="aggregate">A per-sample aggregate (<see cref="Serves"/>).</param>
|
||||
/// <param name="plan">The bucket plan.</param>
|
||||
/// <param name="maxWindowRows">The in-window raw-row cap (<= 0 = unbounded).</param>
|
||||
/// <param name="now">Server timestamp, and the anchor of the look-ahead horizon.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>One value per interval, in the plan's order.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">The window holds more than <paramref name="maxWindowRows"/> rows,
|
||||
/// or the stream could not be read to completion within the requested row budget.</exception>
|
||||
public static async Task<IReadOnlyList<DataValueSnapshot>> ReadAsync(IHistorianGatewayClient client, GatewayRowCeiling ceiling, string tag,
|
||||
HistoryAggregateType aggregate, BucketPlan plan, int maxWindowRows, DateTime now, CancellationToken ct)
|
||||
{
|
||||
if (plan.BucketCount == 0) return [];
|
||||
var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
|
||||
var requestStart = RequestStartFor(plan.StartUtc);
|
||||
|
||||
// A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
|
||||
// the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
|
||||
var requested = MaxValuesFor(maxWindowRows);
|
||||
var (pass, maxValues) = await ceiling.ReadAsync(requested,
|
||||
budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
|
||||
(r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
|
||||
var (buckets, earliest, latest, received, overCap) = pass;
|
||||
if (overCap) throw LimitExceeded(tag, maxWindowRows);
|
||||
if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);
|
||||
|
||||
if (needsRange)
|
||||
{
|
||||
// Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
|
||||
// inside the window).
|
||||
var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
|
||||
if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
|
||||
if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
|
||||
earliest = before;
|
||||
|
||||
// Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
|
||||
// otherwise a small separate read from the end to now + horizon.
|
||||
var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
|
||||
var horizon = DateTime.MaxValue - now > LookAheadHorizon
|
||||
? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
|
||||
: DateTime.MaxValue;
|
||||
if (!laterSeen && horizon > plan.EndUtc)
|
||||
if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
|
||||
latest = after;
|
||||
}
|
||||
|
||||
return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
|
||||
}
|
||||
|
||||
/// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
|
||||
private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
|
||||
ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
|
||||
DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
|
||||
{
|
||||
var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
|
||||
DateTime? earliest = null, latest = null;
|
||||
long windowRows = 0, received = 0;
|
||||
var overCap = false;
|
||||
// Consumed to COMPLETION (never broken out of): abandoning a gateway stream cancels the call server-side, and the
|
||||
// gateway currently treats that cancellation as a connection fault and evicts its shared historian session. Past
|
||||
// the cap the rest is drained (bounded by maxValues) without being evaluated.
|
||||
await foreach (var raw in client.ReadRawAsync(tag, requestStart, plan.EndUtc, maxValues, ct).ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (overCap || raw.Timestamp is null) continue;
|
||||
var s = ToStored(raw);
|
||||
var t = s.TimeUtc;
|
||||
if (!IsAfterFloor(t)) continue; // the boundary row of a window starting at/before the epoch, never a sample
|
||||
if (plan.IndexOf(t) is { } bucket)
|
||||
{
|
||||
if (maxWindowRows > 0 && ++windowRows > maxWindowRows)
|
||||
{
|
||||
overCap = true;
|
||||
continue;
|
||||
}
|
||||
buckets.Add(bucket, s);
|
||||
}
|
||||
// The data range as the SQL backends define it: earliest stored sample at or before the window's later edge,
|
||||
// latest at or after its earlier edge. A row before the window (the AVEVA boundary row, or a stored sample in
|
||||
// the margin) proves earlier data only when its OPC quality is not Bad: AVEVA's boundary row for a tag with no
|
||||
// earlier data is a NULL row, which arrives WITH a value (histsdk sets NumericValue = 0 / StringValue "") and
|
||||
// a Bad OPC quality (captured: 1, no-data).
|
||||
if (t >= plan.StartUtc || IsNotBadOnTheWire(raw))
|
||||
{
|
||||
if (t <= plan.EndUtc && (earliest is null || t < earliest)) earliest = t;
|
||||
if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
|
||||
}
|
||||
}
|
||||
return (buckets, earliest, latest, received, overCap);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The time of the first stored sample (any quality) strictly between <paramref name="startUtc"/> and
|
||||
/// <paramref name="endUtc"/>, read as a <see cref="ProbeRows"/>-row raw read consumed to completion. A row stamped
|
||||
/// exactly at the query start is the historian's boundary row (the value in effect there, or a NULL row when there is
|
||||
/// none) and a row exactly at the query end may be a terminal row — neither is a stored sample, whatever its value.
|
||||
/// </summary>
|
||||
private static async Task<DateTime?> FirstSampleAsync(IHistorianGatewayClient client, string tag, DateTime startUtc,
|
||||
DateTime endUtc, CancellationToken ct)
|
||||
{
|
||||
DateTime? found = null;
|
||||
await foreach (var raw in client.ReadRawAsync(tag, startUtc, endUtc, ProbeRows, ct).ConfigureAwait(false))
|
||||
{
|
||||
if (found is not null || raw.Timestamp is null) continue; // drain, never break (see ReadAsync)
|
||||
var t = raw.Timestamp.ToDateTime();
|
||||
if (t > startUtc && t < endUtc) found = t;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The row's OPC DA quality alone (<c>opc_quality</c> ≥ 64: Good or Uncertain) — deliberately without
|
||||
/// <see cref="SampleMapper.ToStatus"/>'s fallback to the AVEVA <c>quality</c> field when <c>opc_quality</c> is 0, which
|
||||
/// is also the OPC DA Bad value a NULL row carries.
|
||||
/// </summary>
|
||||
private static bool IsNotBadOnTheWire(HistorianSample raw) => (byte)raw.OpcQuality >= 64; // low byte, as SampleMapper.ToStatus reads it
|
||||
|
||||
/// <summary>
|
||||
/// The stream delivered its whole row budget, so the window cannot be proven complete. Names the limit that actually
|
||||
/// bound the budget: the gateway's ceiling (a strict gateway refused the requested budget and named it, #691.3), the
|
||||
/// #688 clamp at the gateway's default ceiling, or the configured cap.
|
||||
/// </summary>
|
||||
private static HistoryReadLimitExceededException BudgetExhausted(string tag, int maxWindowRows, int requested, int budget)
|
||||
{
|
||||
if (budget < requested)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's row ceiling ({budget}, its " +
|
||||
"Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); the Gateway backend evaluates " +
|
||||
"per-sample aggregates from raw reads — narrow the window.");
|
||||
if (maxWindowRows > 0 && budget < (long)maxWindowRows + 2 + PreWindowAllowance)
|
||||
return new HistoryReadLimitExceededException(budget,
|
||||
$"The processed read of '{tag}' covers more raw samples than the HistorianGateway's default row ceiling " +
|
||||
$"({budget}) the request is clamped at; the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
return LimitExceeded(tag, maxWindowRows);
|
||||
}
|
||||
|
||||
private static HistoryReadLimitExceededException LimitExceeded(string tag, int maxWindowRows) =>
|
||||
new(maxWindowRows,
|
||||
$"The processed read of '{tag}' covers more than {maxWindowRows} raw samples " +
|
||||
"(ServerHistorian:MaxTimeWeightedRawRows); the Gateway backend evaluates per-sample aggregates from raw reads — " +
|
||||
"narrow the window.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the per-bucket statistics (<see cref="BucketRow"/>) the SQL backends' aggregate queries produce, from a stream
|
||||
/// of raw samples — the C# statement of <c>SqlAggregateQueries</c>' columns: counts by class (usable = non-Bad with a
|
||||
/// value; good = usable with Good severity; NonNumeric = usable String/DateTime; GoodNonNumeric = good String/DateTime),
|
||||
/// Avg/Min/Max/Sum over the GOOD numeric values and UncertainMin/UncertainMax over the usable-but-not-Good numeric values
|
||||
/// (#684), first/last usable value by time, the extreme GOOD numeric value with its time (ties ⇒ earliest), and the
|
||||
/// first/last sample of any quality.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Keep in step with the SQL backends' per-sample value columns: a change to which samples feed a statistic there
|
||||
/// (e.g. Uncertain handling) must be repeated here, or the Gateway backend silently diverges.
|
||||
/// </remarks>
|
||||
internal sealed class RawBucketAccumulator
|
||||
{
|
||||
private readonly bool _maximum;
|
||||
private readonly Dictionary<long, State> _buckets = new();
|
||||
|
||||
/// <summary>Creates an accumulator.</summary>
|
||||
/// <param name="maximum">Extreme = the maximum (MaximumActualTime); otherwise the minimum.</param>
|
||||
public RawBucketAccumulator(bool maximum) => _maximum = maximum;
|
||||
|
||||
private sealed class State
|
||||
{
|
||||
public int Total, Usable, Good, NonNumeric, GoodNonNumeric, GoodNumeric, UncertainNumeric;
|
||||
public double Sum, Min = double.PositiveInfinity, Max = double.NegativeInfinity;
|
||||
public double UncertainMin = double.PositiveInfinity, UncertainMax = double.NegativeInfinity;
|
||||
public StoredSample? First, Last, FirstAny, LastAny, Extreme;
|
||||
}
|
||||
|
||||
/// <summary>Adds one in-window sample to its bucket. Order-independent (picks compare by time).</summary>
|
||||
/// <param name="bucket">Ascending bucket index (<see cref="BucketPlan.IndexOf"/>).</param>
|
||||
/// <param name="s">The sample.</param>
|
||||
public void Add(long bucket, StoredSample s)
|
||||
{
|
||||
if (!_buckets.TryGetValue(bucket, out var st)) _buckets[bucket] = st = new State();
|
||||
st.Total++;
|
||||
if (st.FirstAny is not { } fa || s.TimeUtc < fa.TimeUtc) st.FirstAny = s;
|
||||
if (st.LastAny is not { } la || s.TimeUtc >= la.TimeUtc) st.LastAny = s;
|
||||
if (!s.IsUsable) return;
|
||||
st.Usable++;
|
||||
if (s.IsGood) st.Good++;
|
||||
if (st.First is not { } f || s.TimeUtc < f.TimeUtc) st.First = s;
|
||||
if (st.Last is not { } l || s.TimeUtc >= l.TimeUtc) st.Last = s;
|
||||
if (!s.Value.TryGetDouble(out var v))
|
||||
{
|
||||
st.NonNumeric++;
|
||||
if (s.IsGood) st.GoodNonNumeric++;
|
||||
return;
|
||||
}
|
||||
if (!s.IsGood)
|
||||
{
|
||||
// #684: an Uncertain value is not part of the value set; it only feeds the Min/Max status rule.
|
||||
st.UncertainNumeric++;
|
||||
if (v < st.UncertainMin) st.UncertainMin = v;
|
||||
if (v > st.UncertainMax) st.UncertainMax = v;
|
||||
return;
|
||||
}
|
||||
st.GoodNumeric++;
|
||||
st.Sum += v;
|
||||
if (v < st.Min) st.Min = v;
|
||||
if (v > st.Max) st.Max = v;
|
||||
if (st.Extreme is not { } e) st.Extreme = s;
|
||||
else
|
||||
{
|
||||
var ev = e.Value.Number;
|
||||
var better = _maximum ? v > ev : v < ev;
|
||||
if (better || (v == ev && s.TimeUtc < e.TimeUtc)) st.Extreme = s;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The non-empty buckets' rows, ascending by bucket index.</summary>
|
||||
/// <returns>The rows.</returns>
|
||||
public IEnumerable<BucketRow> Rows() =>
|
||||
_buckets.OrderBy(kv => kv.Key).Select(kv =>
|
||||
{
|
||||
var st = kv.Value;
|
||||
var hasNumeric = st.GoodNumeric > 0;
|
||||
var hasUncertain = st.UncertainNumeric > 0;
|
||||
return new BucketRow
|
||||
{
|
||||
Bucket = kv.Key,
|
||||
Total = st.Total,
|
||||
Usable = st.Usable,
|
||||
Good = st.Good,
|
||||
Bad = st.Total - st.Usable,
|
||||
NonNumeric = st.NonNumeric,
|
||||
GoodNonNumeric = st.GoodNonNumeric,
|
||||
Avg = hasNumeric ? st.Sum / st.GoodNumeric : null,
|
||||
Min = hasNumeric ? st.Min : null,
|
||||
Max = hasNumeric ? st.Max : null,
|
||||
Sum = hasNumeric ? st.Sum : null,
|
||||
UncertainMin = hasUncertain ? st.UncertainMin : null,
|
||||
UncertainMax = hasUncertain ? st.UncertainMax : null,
|
||||
First = st.First?.Value ?? HistorianValue.Empty,
|
||||
Last = st.Last?.Value ?? HistorianValue.Empty,
|
||||
Extreme = st.Extreme?.Value.Number,
|
||||
ExtremeTime = st.Extreme?.TimeUtc,
|
||||
FirstAny = st.FirstAny,
|
||||
LastAny = st.LastAny,
|
||||
};
|
||||
});
|
||||
}
|
||||
+85
-8
@@ -20,7 +20,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
|
||||
/// </remarks>
|
||||
public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
{
|
||||
private readonly record struct LiveEntry(Guid Id, long Start);
|
||||
// Unreadable = the record at Start could not be decoded (and has already been counted). Such a
|
||||
// record is indexed under a synthetic id so FIFO truncation can pass it like any acked entry.
|
||||
private readonly record struct LiveEntry(Guid Id, long Start, bool Unreadable = false);
|
||||
|
||||
private readonly ManagedLocalStorageDevice _device;
|
||||
private readonly FasterLog _log;
|
||||
@@ -41,6 +43,7 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
private readonly Dictionary<Guid, LinkedListNode<LiveEntry>> _index = new();
|
||||
private long _nextScanAddress; // authoritative logical head; peeks scan from here
|
||||
private long _droppedCount;
|
||||
private long _unreadableCount;
|
||||
|
||||
/// <summary>
|
||||
/// Opens (or recovers) the FasterLog-backed outbox under <paramref name="directory"/>.
|
||||
@@ -94,6 +97,24 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Each undecodable record is counted once — at recovery (constructor scan) or, failing that,
|
||||
/// the first time a peek scans it. Such records are skipped by <see cref="PeekBatchAsync"/>,
|
||||
/// passed by any later FIFO ack, and dropped by a peek once they reach the head, so a corrupt
|
||||
/// record can never pin the head.
|
||||
/// </remarks>
|
||||
public long UnreadableCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_state)
|
||||
{
|
||||
return _unreadableCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask AppendAsync(HistorizationOutboxEntry entry, CancellationToken ct)
|
||||
{
|
||||
@@ -182,21 +203,65 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask<IReadOnlyList<HistorizationOutboxEntry>> PeekBatchAsync(int max, CancellationToken ct)
|
||||
/// <remarks>
|
||||
/// Undecodable records are skipped (and counted once, see <see cref="UnreadableCount"/>), never
|
||||
/// thrown. Unreadable records at the HEAD — ahead of every readable entry — are also removed
|
||||
/// here (truncate + commit), because nothing else ever would: a FIFO ack only passes records
|
||||
/// older than the acked entry, so a corrupt record with no readable entry behind it would
|
||||
/// otherwise pin the head (and the outbox count) forever.
|
||||
/// </remarks>
|
||||
public async ValueTask<IReadOnlyList<HistorizationOutboxEntry>> PeekBatchAsync(int max, CancellationToken ct)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(max);
|
||||
|
||||
var batch = new List<HistorizationOutboxEntry>(Math.Min(max, 64));
|
||||
long? droppedHeadTo = null;
|
||||
lock (_state)
|
||||
{
|
||||
// _live and the scan are both in log order, so a forward cursor pairs each scanned record
|
||||
// with its index node (by start address) in O(n) overall.
|
||||
LinkedListNode<LiveEntry>? cursor = _live.First;
|
||||
using FasterLogScanIterator iter = _log.Scan(_nextScanAddress, _log.TailAddress, recover: false);
|
||||
while (batch.Count < max && iter.GetNext(out byte[] bytes, out int len, out _, out _))
|
||||
while (batch.Count < max && iter.GetNext(out byte[] bytes, out int len, out long address, out long next))
|
||||
{
|
||||
batch.Add(HistorizationOutboxEntrySerializer.Deserialize(bytes.AsSpan(0, len)));
|
||||
while (cursor is not null && cursor.Value.Start < address)
|
||||
cursor = cursor.Next;
|
||||
LinkedListNode<LiveEntry>? node = cursor is not null && cursor.Value.Start == address ? cursor : null;
|
||||
|
||||
if (HistorizationOutboxEntrySerializer.TryDeserialize(bytes.AsSpan(0, len), out HistorizationOutboxEntry? entry))
|
||||
{
|
||||
batch.Add(entry);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Count once: a node flagged at recovery (or by an earlier peek) is not re-counted.
|
||||
// An unindexed record cannot be flagged, so it would count per peek — but only this
|
||||
// process's own (readable) appends are ever unindexed, transiently.
|
||||
if (node is null || !node.Value.Unreadable)
|
||||
{
|
||||
_unreadableCount++;
|
||||
if (node is not null)
|
||||
node.Value = node.Value with { Unreadable = true };
|
||||
}
|
||||
|
||||
if (batch.Count == 0 && node is not null && ReferenceEquals(node, _live.First))
|
||||
{
|
||||
cursor = node.Next;
|
||||
_index.Remove(node.Value.Id);
|
||||
_live.RemoveFirst();
|
||||
_nextScanAddress = next;
|
||||
droppedHeadTo = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<HistorizationOutboxEntry>>(batch);
|
||||
if (droppedHeadTo is long head)
|
||||
{
|
||||
_log.TruncateUntil(head);
|
||||
await _log.CommitAsync(ct).ConfigureAwait(false); // make the head advance durable
|
||||
}
|
||||
|
||||
return batch;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -254,9 +319,21 @@ public sealed class FasterLogHistorizationOutbox : IHistorizationOutbox
|
||||
using FasterLogScanIterator iter = _log.Scan(_log.BeginAddress, _log.TailAddress, recover: false);
|
||||
while (iter.GetNext(out byte[] bytes, out int len, out long currentAddress, out _))
|
||||
{
|
||||
HistorizationOutboxEntry entry = HistorizationOutboxEntrySerializer.Deserialize(bytes.AsSpan(0, len));
|
||||
LinkedListNode<LiveEntry> node = _live.AddLast(new LiveEntry(entry.Id, currentAddress));
|
||||
_index[entry.Id] = node;
|
||||
// An undecodable record must not fail construction (that would take the recorder down
|
||||
// with it): index it under a synthetic id so FIFO truncation can pass it, flag it so
|
||||
// PeekBatchAsync skips it without re-counting, and count it once here.
|
||||
LiveEntry live;
|
||||
if (HistorizationOutboxEntrySerializer.TryDeserialize(bytes.AsSpan(0, len), out HistorizationOutboxEntry? entry))
|
||||
{
|
||||
live = new LiveEntry(entry.Id, currentAddress);
|
||||
}
|
||||
else
|
||||
{
|
||||
live = new LiveEntry(Guid.NewGuid(), currentAddress, Unreadable: true);
|
||||
_unreadableCount++;
|
||||
}
|
||||
|
||||
_index[live.Id] = _live.AddLast(live);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+29
-9
@@ -28,12 +28,18 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Recorder;
|
||||
/// <b>Quality is a documented-dead input</b> on the <c>WriteLiveValues</c> SQL path
|
||||
/// (HistorianGateway 0.2.0 Contracts C-002): <c>HistorianLiveValue.Quality</c> /
|
||||
/// <c>QualityDetail</c> are server-managed columns — any value sent is accepted on the wire and
|
||||
/// then silently discarded. We populate <c>Quality</c> anyway (see the mapping below): proto3
|
||||
/// transmits <c>0</c> when a field is left unset, and <c>0</c> is "Bad" in OPC-DA terms, so
|
||||
/// <em>not</em> setting it would send a <em>wrong</em> quality to any future gateway version that
|
||||
/// starts honoring the field — whereas the recorder's <c>192</c> ("Good") is the correct value.
|
||||
/// <b>Do NOT extend the recorder to capture real node quality expecting it to persist</b> — it
|
||||
/// will not, until the gateway honors the field.
|
||||
/// then silently discarded. We populate <c>Quality</c> anyway: proto3 transmits <c>0</c> when a
|
||||
/// field is left unset, and <c>0</c> is "Bad" in OPC-DA terms, so <em>not</em> setting it would send a
|
||||
/// <em>wrong</em> quality to any future gateway version that starts honoring the field. The recorder
|
||||
/// now carries each sample's real OPC UA status, which <see cref="Mapping.GatewayQualityMapper.ToDaQuality"/>
|
||||
/// maps to the DA byte (Good ⇒ <c>192</c>, as before). <b>That status does not persist on this
|
||||
/// backend</b>: the gateway discards the field, so AVEVA still stores every sample Good — use the
|
||||
/// SqlServer backend where the stored status must be the real one.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Numeric only.</b> Values whose kind has no double form (String, DateTime, Empty) are skipped,
|
||||
/// never thrown; a batch with none left is acked without calling the gateway. The recorder already drops
|
||||
/// those kinds at capture for this backend, so the skip is a backstop.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GatewayHistorianValueWriter : IHistorianValueWriter, IAsyncDisposable
|
||||
@@ -68,12 +74,20 @@ public sealed class GatewayHistorianValueWriter : IHistorianValueWriter, IAsyncD
|
||||
var liveValues = new List<HistorianLiveValue>(values.Count);
|
||||
foreach (HistorizationValue value in values)
|
||||
{
|
||||
if (!value.Value.TryGetDouble(out double number))
|
||||
{
|
||||
// The gateway live path is numeric-only; non-numeric kinds are skipped, never thrown.
|
||||
_logger.LogDebug("Skipping non-numeric {Kind} value for live write.", value.Value.Kind);
|
||||
continue;
|
||||
}
|
||||
|
||||
var live = new HistorianLiveValue
|
||||
{
|
||||
NumericValue = value.Value,
|
||||
NumericValue = number,
|
||||
// Documented-dead input on the WriteLiveValues SQL path (0.2.0 Contracts C-002 —
|
||||
// server-managed, silently discarded). Populated deliberately; see the class <remarks>.
|
||||
Quality = value.Quality,
|
||||
// server-managed, silently discarded). Populated deliberately from the value's
|
||||
// status; Good maps to 192 as before. See the class <remarks>.
|
||||
Quality = Mapping.GatewayQualityMapper.ToDaQuality(value.StatusCode),
|
||||
};
|
||||
|
||||
if (value.TimestampUtc is { } timestampUtc)
|
||||
@@ -86,6 +100,12 @@ public sealed class GatewayHistorianValueWriter : IHistorianValueWriter, IAsyncD
|
||||
liveValues.Add(live);
|
||||
}
|
||||
|
||||
if (liveValues.Count == 0)
|
||||
{
|
||||
// Every value was non-numeric: nothing to send, trivially drained.
|
||||
return true;
|
||||
}
|
||||
|
||||
WriteAck ack = await _client.WriteLiveValuesAsync(tag, liveValues, ct).ConfigureAwait(false);
|
||||
return ack.Success || ack.Queued;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user