diff --git a/.gitignore b/.gitignore index 463f374..680e2bb 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ Thumbs.db # Test droppings *.coverage coverage.cobertura.xml + +# Live 2023 R2 server credentials β€” never commit +wonder-sql-vd03.txt diff --git a/docs/plans/hcal-capability-matrix.md b/docs/plans/hcal-capability-matrix.md index 9dc7e71..234cf2d 100644 --- a/docs/plans/hcal-capability-matrix.md +++ b/docs/plans/hcal-capability-matrix.md @@ -99,7 +99,7 @@ blob needs RE). |---|---|---|---|---|---| | System parameter | `GetSystemParameter` | `Status.GetSystemParameter` | βœ… | DONE | | | Runtime parameter | `GetRuntimeParameter` | `Status.GetRuntimeParameter` | ⬜ | TRIVIAL | same shape as GetSystemParameter | -| Historian info | `GetHistorianInfo` | `Status.GetHistorianInfo` | πŸŸ— | BOUNDED | GETHI buffer; partially decoded (incl. EventStorageMode @ offset 514) | +| Historian info | `GetHistorianInfo` | `Status.GetHistorianInfo` | πŸŸ— | BOUNDED | **2020 WCF = version-only** (GETHI is a named-value query; `EventStorageMode` not on the wire). 518-byte struct + `EventStorageMode`@514 is gRPC/2023R2-only. See `wcf-historian-info.md` | | Server timezone | `GetSystemTimeZoneInfo` | `Status.GetSystemTimeZoneName` | ⬜ | TRIVIAL | | | Historization status | `GetHistorizationStatus` | `Status` op | ⬜ | BOUNDED | | | Store-and-forward status | `GetStoreForwardStatus` | (push events / pull GETHI) | πŸŸ— | HARD | currently synthesized; real read needs duplex push or a decoded pull endpoint β€” see store-forward plan | diff --git a/docs/plans/hcal-roadmap.md b/docs/plans/hcal-roadmap.md index 62b9f86..5df3ba1 100644 --- a/docs/plans/hcal-roadmap.md +++ b/docs/plans/hcal-roadmap.md @@ -41,10 +41,14 @@ HCAL replacement, built on the **2023 R2 gRPC transport**. Derived from > (βœ… 2026-06-20 β€” `ReadEventsAsync(…, HistorianEventFilter)`, live-honored). M2 event send is > also done (βœ… WCF `AddS2`). **R1.2 `GetRuntimeParameterAsync` is also done** (βœ… 2026-06-20, > `aa/Stat/GETRP`, live-verified) β€” notably a *string-handle* op that punches through the wall -> using the Open2 storage-session GUID as an **uppercase** string handle, which is a strong lead -> that the GETHI/ExeC failures are (at least partly) a handle-*format* issue rather than only a -> missing native registration. **Cheap high-value follow-up: retry GETHI/ExeC with the uppercased -> storage GUID** before assuming the registration wall (see `wcf-string-handle-wall.md` Β§Update). +> using the Open2 storage-session GUID as an **uppercase** string handle, which proved the +> GETHI/ExeC failures were a handle-*format* issue rather than a missing native registration. +> **Follow-up done:** R1.1 `ExecuteSqlCommandAsync` shipped; R1.5 extended-property read shipped +> (R1.6 collapsed into it β€” no distinct localized op). **R1.4 `GetHistorianInfo` bounded out on +> 2020 WCF** β€” GETHI there is a named-value query (only `HistorianVersion`); `EventStorageMode` is +> 2023R2-gRPC-only (see `wcf-historian-info.md`). Net: the **reachable 2020-WCF M1 read surface is +> complete**; what remains is config *writes* (M1c β€” gated on an explicit user request) and the +> gRPC/2023R2-only items (R1.3 timezone, R1.4 EventStorageMode β€” need a live 2023 R2 server). ## Guiding principles @@ -90,7 +94,7 @@ read/browse/status surface is Windows-free and the gRPC stack is the default pat ### 1a. Trivial (XS–S each, no new payload format) | ID | Capability | gRPC op | Notes | |---|---|---|---| -| R1.1 | `ExecuteSqlCommandAsync` | `Retrieval.ExecuteSqlCommand` (`ExeC`+`GetR`) | βœ… **REACHABLE (2026-06-20, live-probed).** The earlier "code 51 blocked" verdict was a handle-**format** bug β€” `ExeC` succeeds with the Open2 storage GUID sent **uppercase** (`ToString("D").ToUpperInvariant()`). Chain: `Retr.GetV` prime β†’ `ExeC(handle, sqlString, option=0, ref queryHandle)` β†’ `GetR(handle, queryHandle, ref sequence)` returns the result as a **BinaryFormatter-serialized .NET DataTable**. Proven by `StringHandleProbeDiagnosticTests` + `scripts/Capture-ExecSql.ps1`. **Public API not yet shipped** β€” needs a `GetR` continuation loop + a custom BinaryFormatter-stream parser (BinaryFormatter is removed from .NET 10, so a DataTable can't just be deserialized). | +| ~~R1.1~~ | ~~`ExecuteSqlCommandAsync`~~ | `Retrieval.ExecuteSqlCommand` (`ExeC`+`GetR`) | βœ… **DONE (2026-06-20), live-verified.** `ExecuteSqlCommandAsync(sql)` β†’ `HistorianSqlResult` (columns + typed rows). String-handle op via the uppercase storage GUID. Chain: `Retr.GetV` prime β†’ `ExeC(handle, sql, option=0, ref queryHandle)` β†’ `GetR` loop (note: `GetR` returns **false even on success** β€” the stream is in `pResultBuff` regardless; false = final page). `GetR`'s `pResultBuff` is an **NRBF-serialized `DataTable`** (`SerializationFormat.Xml`: members `XmlSchema` + `XmlDiffGram`). BinaryFormatter is gone from .NET 10, so it's decoded read-only with `System.Formats.Nrbf` + `XDocument` (no BinaryFormatter). Shipped: `HistorianSqlResult`/`HistorianSqlColumn`/`HistorianSqlExecuteOption`, `HistorianSqlResultProtocol`, `HistorianWcfSqlClient`, golden `WcfSqlResultProtocolTests`, gated live tests. See `docs/reverse-engineering/wcf-exec-sql.md`. | | ~~R1.2~~ | ~~`GetRuntimeParameterAsync`~~ | `Status.GetRuntimeParameter` (`aa/Stat/GETRP`) | βœ… **DONE (2026-06-20), live-verified.** Captured (`scripts/Capture-RuntimeParam.ps1`): GETRP is a **`string`-handle** op (GETHI's shape), but reachable from the managed client using the Open2 storage-session GUID as an **uppercase** string handle (`ToString("D").ToUpperInvariant()`). Returns `HistorianVersion` = `20,0,000,000` live. pRequestBuff = `54 67 01 00` + uint nameCount + per-name(uint charCount + UTF-16); pResponseBuff = version + uint resultCount + CRetVariant(`0x43` VT_BSTR + uint16 len + uint16 charCount + UTF-16). Single string-valued param only (multi-name framing inferred, not captured). Shipped: `HistorianClient.GetRuntimeParameterAsync(name)`; golden `WcfRuntimeParameterProtocolTests`. **Note:** GETRP punching through the string-handle wall with the uppercase storage GUID is a strong lead that GETHI/ExeC may be a handle-*format* issue β€” see `wcf-string-handle-wall.md` Β§Update. | | ~~R1.3~~ | ~~`GetServerTimeZoneAsync`~~ | `Status.GetSystemTimeZoneName` | β›” **gRPC/2023R2-only β€” re-confirmed 2026-06-21 from 3 angles.** (1) native `GetSystemTimeZoneName` is a stub (rc=0, empty) in the `GetServerTime` family; (2) Runtime DB has no timezone `SystemParameter` β€” the zone exists only as per-block `HistoryBlock.TimeZoneOffset`/`wwTimeZone` (DST-specific, SQL-only) + a `TimeZone` lookup table, `StorageShard.TimeZoneId`=NULL; (3) `GetSystemParameter` + `GETRP` return null/throw for every timezone candidate (only `HistorianVersion` works). The sole 2020 route is a SQL read via `ExecuteSqlCommand` (R1.1) β€” DST-specific, different mechanism. Build the real op only against a live 2023 R2 server. See `docs/reverse-engineering/wcf-status-localhost.md`. | diff --git a/docs/reverse-engineering/wcf-exec-sql.md b/docs/reverse-engineering/wcf-exec-sql.md new file mode 100644 index 0000000..263e0d9 --- /dev/null +++ b/docs/reverse-engineering/wcf-exec-sql.md @@ -0,0 +1,86 @@ +# SQL command execution over 2020 WCF β€” ExeC + GetR (HCAL R1.1) + +**Status: βœ… DONE + live-verified (2026-06-20).** `HistorianClient.ExecuteSqlCommandAsync(sql)` runs a +SQL command against the Historian over the 2020 WCF ops `aa/Retr/ExeC` (ExecuteSqlCommand) + +`aa/Retr/GetR` (GetRecordSetByteStream) and returns the record set as a `HistorianSqlResult` (the +managed equivalent of the native `DataTable`). Live-verified end-to-end from the pure-managed .NET 10 +client against the local 2020 Historian (single-cell, multi-column/multi-row, and NULL cases). + +## The ops + +Both are on the Retrieval service (`IRetrievalServiceContract3`), **string-handle** ops reached with +the Open2 storage-session GUID formatted `storageSessionId.ToString("D").ToUpperInvariant()` (the same +uppercase handle that unlocked GETRP/GETHI; see `wcf-string-handle-wall.md`). `Retr.GetV` is primed +first. + +``` +bool ExecuteSqlCommand(string handle, string command, uint option, + ref uint queryHandle, out int retValue, out uint errorSize, out byte[] errorBuffer) + +bool GetRecordSetByteStream(string handle, uint queryHandle, ref uint sequence, + out uint resultSize, out byte[] pResultBuff, out uint errorSize, out byte[] errorBuffer) +``` + +- **`command`** is sent as a plain string (MDAS-encoded ASCII), e.g. `SELECT 1 AS ProbeValue`. +- **`option`** = `HistorianSqlExecuteOption` (`ExecuteRecord=0` is the captured/proven record-set path). +- **`ExeC`** returns the assigned `queryHandle` (and `retValue`); pass it to `GetR`. +- **`GetR` returns `false` even on success** β€” the byte stream is in `pResultBuff` regardless; a + `false` result just signals the final page. So the orchestrator always consumes `pResultBuff`, then + stops on a `false` result or an empty page. (`sequence` is the paging cursor; small record sets + return everything in one call.) + +## The result stream β€” NRBF-wrapped DataTable (no BinaryFormatter) + +`GetR`'s `pResultBuff` is a **.NET Remoting Binary Format (NRBF)** stream wrapping a +`System.Data.DataTable` serialized with `SerializationFormat.Xml`. Stream shape (captured): + +``` +SerializationHeader (00 01 00 00 00 FF FF FF FF 01 00 00 00 00 00 00 00) +BinaryLibrary (0C): "System.Data, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" +ClassWithMembersAndTypes (05): System.Data.DataTable, members: + DataTable.RemotingVersion -> System.Version object + XmlSchema -> BinaryObjectString (XSD: column names + xs types) + XmlDiffGram -> BinaryObjectString (ADO.NET diffgram: the rows) +System.Version object (_Major/_Minor/_Build/_Revision) +MessageEnd (0B) +``` + +**BinaryFormatter is removed from .NET 10**, so the stream is decoded read-only with +`System.Formats.Nrbf.NrbfDecoder` (the sanctioned successor β€” it parses records without instantiating +or executing any payload type; added as a managed `PackageReference`). The two embedded XML strings are +then parsed with `XDocument`: + +- **XmlSchema** β†’ columns (name + XSD type) under the `msdata:MainDataTable` element's sequence. +- **XmlDiffGram** β†’ rows (each row element under the dataset; cells are child elements or attributes + matching column names). NULL cells are simply absent β†’ parsed as `null`. + +Values are typed per the XSD type (`xs:int`β†’int, `xs:string`β†’string, `xs:dateTime`β†’DateTime, …), +falling back to the raw string for unrecognized types. Only the `SerializationFormat.Xml` DataTable +shape is evidence-backed; a stream whose root is not a DataTable class record, or that lacks the two +XML members, throws `ProtocolEvidenceMissingException`. + +## Capture / decode tooling + +`scripts/Capture-ExecSql.ps1` (NativeTraceHarness `exec-sql` scenario + instrument-wcf-{write,read}message) +captures the ExeC/GetR exchange. ⚠️ A **raw** instrument-wcf capture interleaves MDAS transport chunk +markers (`0x9F`/`0x9E`) into a large `pResultBuff`, so raw byte-slicing yields a corrupted NRBF stream. +The **clean** contract-level byte[] (what the WCF channel reassembles) is dumped via the +`AVEVA_HISTORIAN_SQL_DUMP` env var on `HistorianWcfSqlClient` β€” that is the golden fixture in +`WcfSqlResultProtocolTests` (the benign `SELECT 1 AS ProbeValue`, no sensitive data). + +## Shipped surface + +- `HistorianClient.ExecuteSqlCommandAsync(command, option = ExecuteRecord)` β†’ `HistorianSqlResult` + (`Columns` name+type, `Rows` typed values, `ReturnValue`). +- Models `HistorianSqlResult` / `HistorianSqlColumn` / `HistorianSqlExecuteOption`; + `HistorianSqlResultProtocol` (NRBF + diffgram parser); `HistorianWcfSqlClient` (ExeC/GetR + orchestration); golden `WcfSqlResultProtocolTests`; gated live tests + (`ExecuteSqlCommandAsync_AgainstLocalHistorian_ReturnsRecordSet` and `_MultiColumnMultiRow`). + +## Scope notes + +- `ExecuteRecord` (record set) is the evidence-backed path. `ExecuteNonQuery`/`ExecuteScalar`/ + `ExecuteRecordDirect` are accepted via the option enum but their non-record-set return shapes are not + separately captured β€” a non-record result yields empty `Columns`/`Rows` with the `ReturnValue` set. +- The command is whatever the Historian's SQL surface accepts (it routes to the Runtime DB). No + client-side SQL validation is performed. diff --git a/docs/reverse-engineering/wcf-historian-info.md b/docs/reverse-engineering/wcf-historian-info.md new file mode 100644 index 0000000..7efeeb6 --- /dev/null +++ b/docs/reverse-engineering/wcf-historian-info.md @@ -0,0 +1,68 @@ +# GetHistorianInfo over 2020 WCF β€” GETHI is named-value-only (HCAL R1.4) + +**Status: ⚠ Bounded out on 2020 WCF (2026-06-20).** `GetHistorianInfoAsync` is **not shipped**: +the one field that motivates it β€” `EventStorageMode` β€” is **not on the 2020 WCF wire**. The +version field that GETHI *does* return over WCF is already exposed (`ProbeAsync`, +`GetRuntimeParameterAsync("HistorianVersion")`), so there is nothing new to ship here without a +2023 R2 gRPC server. This parallels R1.3 (`GetServerTimeZone`), which is likewise 2023R2-only. + +## What the capture showed + +`scripts/Capture-HistorianInfo.ps1` drives the native `HistorianAccess.GetHistorianInfo(out +HistorianInfo, out error)` through the instrumented (`instrument-wcf-{write,read}message`) +`current/aahClientManaged.dll`. The native call **succeeds** and returns +`EventStorageMode = Blocks`, `ServerVersion = 20,0,000,000`, no error. + +But the wire tells a different story (`scripts/decode-historian-info-capture.py`): + +- The only `GETHI` op on the wire is **`aa/Stat/GETHI(handle, pRequestBuff)`** with + `pRequestBuff = 53 67 02 00` (sig `0x6753` + version `2`) `+ uint charCount(16) + UTF-16 + "HistorianVersion"` β€” i.e. the **named-value request**, identical to the GETRP/version shape. +- Its response `pResponseBuff` is **~30 bytes**: `uint charCount(12) + UTF-16 "20,0,000,000"` + (+ a `02 00 01 00` trailer). **Just the version** β€” not a 518-byte struct. +- The post-GETHI ops in the same capture are `Hist/UpdC3` + a run of `Stat/GetSystemParameter` + (`AllowOriginals`, `HistorianPartner`, `HistorianVersion`, `MaxCyclicStorageTimeout`, + `RealTimeWindow`, `FutureTimeThreshold`, `AllowRenameTags`). **None carries a storage-mode + value.** So the native wrapper's `EventStorageMode` is derived by the C++ HCAL **outside the + WCF wire**, not fetched over it. + +## Probe: does GETHI expose storage mode under any name? + +`StringHandleProbeDiagnosticTests.GETHI_CandidateInfoNames_AgainstLocalHistorian` (gated on +`HISTORIAN_HOST=localhost`) issues GETHI for `HistorianVersion` plus seven storage-mode name +guesses. Result on the live 2020 server: + +| GETHI parameter name | result | +|---|---| +| `HistorianVersion` | **ok=True**, respLen=32 (version) | +| `EventStorageMode`, `EventStorageType`, `StorageType`, `HistorianEventStorageMode`, `EventStorage`, `StorageMode`, `HistorianInfo` | **ok=False**, errLen=5, empty | + +So GETHI on 2020 WCF is a strict named-value lookup with exactly one known-good key +(`HistorianVersion`). There is no storage-mode key, no full-struct request. + +## Why the 518-byte struct doesn't apply here + +The 2023 R2 decompiled `ArchestrA.HistorianAccess.GetHistorianInfo` (analysis folder) allocates +a **518-byte `HISTORIAN_INFO`** struct, pre-inits `int32 @514` to `-1`, calls native HCAL +(vtable+648) which fills it, then reads version (UTF-16 @0) + `EventStorageMode` (`@514`: +`-1`=Unsupported, `0`=Database, else=Blocks). That is the **HCAL-native / 2023R2 gRPC** +front-door model (`StatusService.GetHistorianInfo` returns `bytes btHistorianInfo`). On **2020 +WCF** that struct is never marshaled across the wire β€” only the version named-value is. The +native client's `EventStorageMode` therefore comes from C++-internal state the managed WCF +replay cannot observe or reproduce. + +## Conclusion / where it lands + +- **2020 WCF:** `GetHistorianInfoAsync` would add nothing over existing surface (version only) and + could not report a real `EventStorageMode` β€” so it is intentionally **not shipped** (no hollow + `Unsupported`-returning API; project discipline: don't ship misleading behavior). +- **2023 R2 gRPC:** `Status.GetHistorianInfo` returns the full 518-byte `btHistorianInfo`; decode + version@0 + `EventStorageMode`@514 there. Build + verify against a live 2023 R2 server. The + `HistorianInfo` / `HistorianEventStorageMode` public types should land alongside that path. + +## Tooling kept as RE aids + +- `tools/AVEVA.Historian.NativeTraceHarness` `historian-info` scenario (drives the native call). +- `scripts/Capture-HistorianInfo.ps1` + `scripts/decode-historian-info-capture.py`. +- `StringHandleProbeDiagnosticTests.GETHI_CandidateInfoNames_AgainstLocalHistorian` (locks the + named-value-only finding; gated). diff --git a/docs/reverse-engineering/wcf-string-handle-wall.md b/docs/reverse-engineering/wcf-string-handle-wall.md index 34b2122..1e94dcd 100644 --- a/docs/reverse-engineering/wcf-string-handle-wall.md +++ b/docs/reverse-engineering/wcf-string-handle-wall.md @@ -110,14 +110,28 @@ GETHI probe is the **handle string format**: the native client sends the GUID lowercase, so a probe that passed the GUID without upcasing would not byte-match what the server's session table is keyed on. -**Implication / open lead (not yet retested):** the GETHI/ExeC/QTB/QTG family failures -may be (at least partly) a **handle-format** issue, not (only) a missing native -registration step. The highest-value cheap follow-up is to **re-probe GETHI and ExeC -with the uppercased storage-session GUID** before assuming the registration wall. If -they also return data, the "wall" collapses to a formatting bug and R1.4/R1.5/R1.6/R1.1 -may be reachable without any new RE. This has **not** been done yet β€” do not reclassify -those items until it is. GETRP is shipped because it was directly captured + live-verified -end-to-end; the rest remain `ProtocolEvidenceMissingException`/unprobed until tested. +**Implication β€” CONFIRMED, the wall is largely a handle-format bug.** The follow-up was done: +GETHI and **ExeC both return data with the uppercased storage-session GUID**. + +- **R1.1 `ExecuteSqlCommandAsync` (ExeC + GetR) β€” SHIPPED + live-verified (2026-06-20).** + `ExecuteSqlCommandAsync(sql)` β†’ `HistorianSqlResult`. `Retr.GetV` prime β†’ `ExeC(handle, + sql, option=0, ref queryHandle)` β†’ `GetR` loop. Note: **`GetR` returns false even on + success** (the byte stream is in `pResultBuff` regardless; false = final page). `pResultBuff` + is an **NRBF `DataTable`** (`SerializationFormat.Xml`: `XmlSchema` + `XmlDiffGram`), decoded + read-only with `System.Formats.Nrbf` + `XDocument` (BinaryFormatter is gone from .NET 10). + Shipped: `HistorianSqlResultProtocol`, `HistorianWcfSqlClient`, golden `WcfSqlResultProtocolTests`, + gated live tests. See `docs/reverse-engineering/wcf-exec-sql.md`. +- **GETHI (R1.4)** returns data with the uppercase handle, **but only the named `HistorianVersion` + value** β€” over 2020 WCF GETHI is a named-value query (the only working key), *not* a full-struct + read. `EventStorageMode` (the 518-byte-struct `@514` field) is **not on the 2020 WCF wire**; it is + the 2023R2 HCAL-native/gRPC model. So R1.4 is **bounded out on WCF / gRPC-2023R2-only** and the + public API is intentionally not shipped. Full analysis: `docs/reverse-engineering/wcf-historian-info.md`. + +So the "wall" collapses to the handle **format** for the Retrieval/Status string-handle ops. +**Exception β€” QTB/QTG:** `StartTagQuery` does *not* punch through; captured with a correctly +uppercase handle it still fails `error 1` **server-side** (`CMdServer::StartActiveTagnamesQuery` +over `\\.\pipe\aahMetadataServer\console`) β€” a metadata-server blocker, independent of handle +format. Name-based ops route around it. See `HistorianRuntimeParameterProtocol`, `IStatusServiceContract2.GetRuntimeParameter`, golden `WcfRuntimeParameterProtocolTests`, and capture tooling diff --git a/scripts/Capture-HistorianInfo.ps1 b/scripts/Capture-HistorianInfo.ps1 new file mode 100644 index 0000000..1fa621d --- /dev/null +++ b/scripts/Capture-HistorianInfo.ps1 @@ -0,0 +1,102 @@ +<# +.SYNOPSIS + Captures the native AVEVA client's GetHistorianInfo wire traffic (HCAL roadmap R1.4) + so the WCF GETHI request that returns the FULL HISTORIAN_INFO struct can be decoded + instead of guessed. + +.DESCRIPTION + Drives the .NET-Framework NativeTraceHarness's `historian-info` scenario against the live + Historian with an IL-rewritten copy of aahClientManaged.dll whose + ClientMessageEncoder.WriteMessage AND ReadMessage are instrumented to log every MDAS body + (the same pipeline that produced every other proven request/response shape). The harness + opens a normal authenticated process connection and calls + HistorianAccess.GetHistorianInfo(out HistorianInfo, out err). + + Decode with scripts/decode-historian-info-capture.py: locate the WCF.WriteMessage.Body + whose op is GETHI -> that is the GetHistorianInfo request; read off the leading string + handle and the pRequestBuff layout (distinct from the named-value "HistorianVersion" + request). The paired WCF.ReadMessage.Body is the pResponseBuff = the 518-byte + HISTORIAN_INFO struct (version string @0 UTF-16 null-terminated, EventStorageMode int32 @514). + +.NOTES + Read-only status call; no data is written. Artifacts are diagnostic and gitignored. + Sanitize before copying anything into docs/ -- never commit raw capture NDJSON, + credentials, hostnames, or customer tag names. +#> +[CmdletBinding()] +param( + [string]$ServerName = "localhost", + [int]$TcpPort = 32568, + [string]$Configuration = "Debug" +) + +$ErrorActionPreference = "Stop" +$repoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $repoRoot + +$reProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseEngineering\AVEVA.Historian.ReverseEngineering.csproj" +$harnessProj = Join-Path $repoRoot "tools\AVEVA.Historian.NativeTraceHarness\AVEVA.Historian.NativeTraceHarness.csproj" +$instrProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\AVEVA.Historian.ReverseInstrumentation.csproj" + +$captureDir = Join-Path $repoRoot "artifacts\reverse-engineering\instrumented-wcf-historian-info" +$currentCopy = Join-Path $captureDir "current-copy" +$instrDll = Join-Path $captureDir "aahClientManaged.dll" +$capturePath = Join-Path $captureDir "historian-info-capture-latest.ndjson" + +Write-Host "== Building tooling ($Configuration) ==" -ForegroundColor Cyan +dotnet build $reProj -c $Configuration --nologo -v q | Out-Null +dotnet build $instrProj -c $Configuration --nologo -v q | Out-Null +dotnet build $harnessProj -c $Configuration --nologo -v q | Out-Null + +$instrSourceDll = Get-ChildItem -Recurse (Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\bin\$Configuration") ` + -Filter "AVEVA.Historian.ReverseInstrumentation.dll" | Select-Object -First 1 -ExpandProperty FullName +if (-not $instrSourceDll) { throw "ReverseInstrumentation.dll not found under bin\$Configuration." } + +Write-Host "== Instrumenting WriteMessage + ReadMessage ==" -ForegroundColor Cyan +New-Item -ItemType Directory -Force -Path $captureDir | Out-Null +# Chain via a distinct intermediate file (reading+writing the same path drops the second +# hook on the mixed-mode native image). Final dll carries both hooks with distinct Phase +# strings: WCF.WriteMessage.Body and WCF.ReadMessage.Body. +$writeOnly = Join-Path $captureDir "aahClientManaged.write.dll" +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-writemessage (Join-Path $repoRoot "current\aahClientManaged.dll") $writeOnly | Out-Null +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-readmessage $writeOnly $instrDll | Out-Null + +Write-Host "== Staging current-copy ==" -ForegroundColor Cyan +robocopy (Join-Path $repoRoot "current") $currentCopy /MIR /NJH /NJS /NDL /NP /NC /NS | Out-Null +Copy-Item -Force $instrDll (Join-Path $currentCopy "aahClientManaged.dll") +Copy-Item -Force $instrSourceDll (Join-Path $currentCopy "AVEVA.Historian.ReverseInstrumentation.dll") + +$harnessDll = Join-Path $currentCopy "aahClientManaged.dll" +if (Test-Path $capturePath) { Remove-Item -Force $capturePath } +$env:AVEVA_HISTORIAN_RE_CAPTURE = $capturePath + +Write-Host "== Capturing historian-info ==" -ForegroundColor Green +$harnessArgs = @( + "--scenario", "historian-info", + "--server-name", $ServerName, + "--tcp-port", "$TcpPort", + "--current-dir", $currentCopy, + "--managed-dll-path", $harnessDll +) + +$harnessJson = $null +try { + $prevEap = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $harnessJson = & dotnet run --no-build -c $Configuration --project $harnessProj -- @harnessArgs 2>&1 +} catch { + Write-Host " (historian-info raised: $($_.Exception.Message))" -ForegroundColor Yellow +} finally { + $ErrorActionPreference = $prevEap +} + +Remove-Item Env:\AVEVA_HISTORIAN_RE_CAPTURE -ErrorAction SilentlyContinue + +$recCount = if (Test-Path $capturePath) { (Get-Content $capturePath | Where-Object { $_.Trim() }).Count } else { 0 } +Write-Host "`n== Capture summary ==" -ForegroundColor Cyan +Write-Host " -> $recCount records -> $capturePath" +Write-Host "Harness output (GetHistorianInfoReturned / HistorianInfo):" -ForegroundColor Cyan +$harnessJson | Select-Object -Last 24 +Write-Host "`nDecode with: python scripts\decode-historian-info-capture.py" -ForegroundColor Cyan diff --git a/scripts/decode-historian-info-capture.py b/scripts/decode-historian-info-capture.py new file mode 100644 index 0000000..b7f55de --- /dev/null +++ b/scripts/decode-historian-info-capture.py @@ -0,0 +1,142 @@ +"""Decode the GetHistorianInfo (GETHI) WCF request/response (HCAL R1.4). + +Reads the chained WriteMessage+ReadMessage capture produced by +scripts/Capture-HistorianInfo.ps1 and locates the GetHistorianInfo exchange. The goal is +to learn (a) the pRequestBuff that returns the FULL HISTORIAN_INFO struct (distinct from the +named-value "HistorianVersion" request) and (b) the response struct layout: the analysis +folder says it's 518 bytes with the version string (UTF-16, null-terminated) at offset 0 and +EventStorageMode (int32) at offset 514. + +We flag candidate bodies by the GETHI op action, by the server version value, and by a +response length near 518, then dump bytes + the int32 at offset 514 so the layout can be +read off directly. + +Output is diagnostic. Sanitize before copying into docs/. +""" +import base64 +import json +import struct +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CAPDIR = REPO_ROOT / "artifacts" / "reverse-engineering" / "instrumented-wcf-historian-info" +CAP = CAPDIR / "historian-info-capture-latest.ndjson" + +# The GETHI op action (WS-Addressing) the native client sends. The server version value is +# version-shaped, not secret; used only to locate the response. +OP_ASCII = b"GetHistorianInfo" +OP_GETHI = b"GETHI" +VERSION = "20,0,000,000" +VERSION_U16 = VERSION.encode("utf-16-le") +VERSION_ASCII = VERSION.encode("ascii") + + +def hexdump(label, buf, base=0): + print(f"=== {label}: {len(buf)} bytes ===") + for off in range(0, len(buf), 16): + c = buf[off:off + 16] + hp = " ".join(f"{x:02X}" for x in c) + ap = "".join(chr(x) if 32 <= x < 127 else "." for x in c) + print(f" {base + off:04X} {hp:<48} |{ap}|") + print() + + +def ascii_strings(buf, minlen=3): + out, cur, start = [], [], 0 + for i, x in enumerate(buf): + if 32 <= x < 127: + if not cur: + start = i + cur.append(chr(x)) + else: + if len(cur) >= minlen: + out.append((start, "".join(cur))) + cur = [] + if len(cur) >= minlen: + out.append((start, "".join(cur))) + return out + + +def u16_strings(buf, minlen=3): + out, i = [], 0 + while i < len(buf) - 1: + j, chars = i, [] + while j < len(buf) - 1 and 32 <= buf[j] < 127 and buf[j + 1] == 0: + chars.append(chr(buf[j])) + j += 2 + if len(chars) >= minlen: + out.append((i, "".join(chars))) + i = j + else: + i += 1 + return out + + +def main() -> int: + if not CAP.exists(): + print(f"Missing capture: {CAP}\nRun scripts/Capture-HistorianInfo.ps1 first.") + return 1 + + records = [] + for line in CAP.open(encoding="utf-8-sig"): + if line.strip(): + records.append(json.loads(line)) + + print(f"== {len(records)} MDAS bodies captured ==") + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + flags = [] + if OP_ASCII in body or OP_GETHI in body: + flags.append("GETHI-OP") + if VERSION_U16 in body or VERSION_ASCII in body: + flags.append("VERSION") + # A ~518-byte embedded struct is the tell for the full-info response. + if 500 <= len(body) <= 4096: + flags.append(f"len={len(body)}") + print(f" [{idx:02d}] {rec.get('Phase'):26s} len={len(body):5d} {','.join(flags)}") + + def find(predicate): + hits = [] + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + if predicate(rec, body): + hits.append((idx, rec, body)) + return hits + + print("\n== Request candidate(s): WriteMessage bodies tagged GETHI-OP ==") + for idx, rec, body in find(lambda r, b: r.get("Phase") == "WCF.WriteMessage.Body" + and (OP_ASCII in b or OP_GETHI in b)): + hexdump(f"[{idx}] WriteMessage", body) + print(" UTF-16 strings:") + for off, s in u16_strings(body): + print(f" 0x{off:04X} {s!r}") + print(" ASCII strings:") + for off, s in ascii_strings(body): + print(f" 0x{off:04X} {s!r}") + print() + + print("\n== Response candidate(s): ReadMessage bodies carrying VERSION ==") + for idx, rec, body in find(lambda r, b: r.get("Phase") == "WCF.ReadMessage.Body" + and (VERSION_U16 in b or VERSION_ASCII in b)): + hexdump(f"[{idx}] ReadMessage", body) + print(" UTF-16 strings:") + for off, s in u16_strings(body): + print(f" 0x{off:04X} {s!r}") + # The analysis folder pins EventStorageMode @ offset 514 (int32) inside the + # 518-byte struct. The struct is embedded in the MDAS body at some base; scan for + # a plausible version@0 run and print the int32 514 bytes after each candidate base. + print(" Candidate struct decodes (version@base, int32 @ base+514):") + for base_off, s in u16_strings(body): + if any(ch.isdigit() for ch in s) and "," in s: + idx514 = base_off + 514 + if idx514 + 4 <= len(body): + mode = struct.unpack_from(" + diff --git a/src/AVEVA.Historian.Client/HistorianClient.cs b/src/AVEVA.Historian.Client/HistorianClient.cs index 7a07107..82c2631 100644 --- a/src/AVEVA.Historian.Client/HistorianClient.cs +++ b/src/AVEVA.Historian.Client/HistorianClient.cs @@ -210,6 +210,22 @@ public sealed class HistorianClient : IAsyncDisposable // documented-but-blocked path in HistorianWcfTagWriteOrchestrator and // docs/reverse-engineering/wcf-add-tag-extended-properties.md Β§Delete. + /// + /// Executes a SQL command against the Historian over the WCF ExeC/GetR ops and + /// returns the record set as a (the managed equivalent of the + /// native DataTable). The record-set path (, + /// the default) is the evidence-backed surface; the result is decoded from the server's + /// NRBF-serialized DataTable without BinaryFormatter. See HistorianSqlResultProtocol. + /// + public Task ExecuteSqlCommandAsync( + string command, + HistorianSqlExecuteOption option = HistorianSqlExecuteOption.ExecuteRecord, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(command); + return _protocol.ExecuteSqlCommandAsync(command, option, cancellationToken); + } + /// /// Creates or updates the named tag in the Historian Runtime database via /// EnsureTags2. Currently only is diff --git a/src/AVEVA.Historian.Client/Models/HistorianSqlExecuteOption.cs b/src/AVEVA.Historian.Client/Models/HistorianSqlExecuteOption.cs new file mode 100644 index 0000000..d249eb1 --- /dev/null +++ b/src/AVEVA.Historian.Client/Models/HistorianSqlExecuteOption.cs @@ -0,0 +1,21 @@ +namespace AVEVA.Historian.Client.Models; + +/// +/// Execution mode for ExecuteSqlCommandAsync, mirroring the native +/// HistorianSqlExecuteOption enum passed to the ExeC op. Only +/// (the record-set path) is evidence-backed end-to-end. +/// +public enum HistorianSqlExecuteOption +{ + /// Execute and return a record set (the captured/proven path). + ExecuteRecord = 0, + + /// Execute without returning a record set; the result carries only the return value. + ExecuteNonQuery = 1, + + /// Execute and return a single scalar value. + ExecuteScalar = 2, + + /// Execute a record set directly (server-side direct path). + ExecuteRecordDirect = 3, +} diff --git a/src/AVEVA.Historian.Client/Models/HistorianSqlResult.cs b/src/AVEVA.Historian.Client/Models/HistorianSqlResult.cs new file mode 100644 index 0000000..c811bb4 --- /dev/null +++ b/src/AVEVA.Historian.Client/Models/HistorianSqlResult.cs @@ -0,0 +1,35 @@ +namespace AVEVA.Historian.Client.Models; + +/// +/// A column in a : its name and the XSD type from the result-set +/// schema (e.g. xs:int, xs:string, xs:dateTime). +/// +public sealed record HistorianSqlColumn(string Name, string SchemaType); + +/// +/// The result of ExecuteSqlCommandAsync β€” the managed equivalent of the DataTable +/// the native client returns. Columns carry name + XSD type; each row is a list of values aligned +/// to (typed per the schema where the XSD type is recognized, otherwise the +/// raw string; null for a SQL NULL / absent cell). +/// +public sealed class HistorianSqlResult +{ + public HistorianSqlResult( + IReadOnlyList columns, + IReadOnlyList> rows, + int returnValue) + { + Columns = columns; + Rows = rows; + ReturnValue = returnValue; + } + + /// The result-set columns, in order. + public IReadOnlyList Columns { get; } + + /// The rows; each row's values align positionally to . + public IReadOnlyList> Rows { get; } + + /// The native retValue from ExeC (e.g. rows affected for non-queries). + public int ReturnValue { get; } +} diff --git a/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs b/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs index 9deb2e7..d3ae8af 100644 --- a/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs +++ b/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs @@ -81,6 +81,13 @@ internal sealed class Historian2020ProtocolDialect return Wcf.HistorianWcfTagExtendedPropertyClient.GetTagExtendedPropertiesAsync(_options, tag, cancellationToken); } + public Task ExecuteSqlCommandAsync(string command, HistorianSqlExecuteOption option, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + return Wcf.HistorianWcfSqlClient.ExecuteSqlCommandAsync(_options, command, option, cancellationToken); + } + private static async IAsyncEnumerable Missing( string operation, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianSqlResultProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianSqlResultProtocol.cs new file mode 100644 index 0000000..50fc7f2 --- /dev/null +++ b/src/AVEVA.Historian.Client/Wcf/HistorianSqlResultProtocol.cs @@ -0,0 +1,221 @@ +using System.Formats.Nrbf; +using System.Globalization; +using System.Xml.Linq; +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Protocol; + +namespace AVEVA.Historian.Client.Wcf; + +/// +/// Parses the GetR (GetRecordSetByteStream) result buffer returned by the 2020 WCF +/// ExeC/GetR SQL surface into a . +/// +/// +/// Captured from the native client (scripts/Capture-ExecSql.ps1 + +/// instrument-wcf-{write,read}message; golden-pinned in WcfSqlResultProtocolTests). The +/// GetR pResultBuff is a .NET Remoting Binary Format (NRBF) stream wrapping a +/// System.Data.DataTable serialized with SerializationFormat.Xml: its data lives in +/// two string members β€” XmlSchema (an XSD describing the columns + types) and +/// XmlDiffGram (an ADO.NET diffgram carrying the rows). +/// +/// BinaryFormatter is removed from .NET 10, so the stream is decoded read-only with +/// (the sanctioned successor β€” it parses records without instantiating +/// or executing any payload type). The two embedded XML strings are then parsed with +/// . +/// +/// Only the SerializationFormat.Xml DataTable shape is evidence-backed; a stream whose +/// root is not a DataTable class record, or that lacks the two XML members, throws +/// . +/// +internal static class HistorianSqlResultProtocol +{ + private const string DataTableTypeName = "System.Data.DataTable"; + private const string XmlSchemaMember = "XmlSchema"; + private const string XmlDiffGramMember = "XmlDiffGram"; + + private static readonly XNamespace Xsd = "http://www.w3.org/2001/XMLSchema"; + private static readonly XNamespace MsData = "urn:schemas-microsoft-com:xml-msdata"; + private static readonly XNamespace DiffGr = "urn:schemas-microsoft-com:xml-diffgram-v1"; + + public static HistorianSqlResult Parse(ReadOnlyMemory resultBuffer, int returnValue) + { + (string? schemaXml, string? diffGramXml) = ExtractXmlMembers(resultBuffer); + + if (string.IsNullOrEmpty(schemaXml)) + { + // No record set (e.g. a non-query, or an empty stream): return columns/rows empty. + return new HistorianSqlResult([], [], returnValue); + } + + (string? tableName, List columns) = ParseSchema(schemaXml); + IReadOnlyList> rows = ParseDiffGram(diffGramXml, tableName, columns); + return new HistorianSqlResult(columns, rows, returnValue); + } + + private static (string? schemaXml, string? diffGramXml) ExtractXmlMembers(ReadOnlyMemory resultBuffer) + { + if (resultBuffer.Length == 0) + { + return (null, null); + } + + SerializationRecord root; + try + { + using MemoryStream stream = new(resultBuffer.ToArray(), writable: false); + root = NrbfDecoder.Decode(stream); + } + catch (Exception ex) when (ex is not ProtocolEvidenceMissingException) + { + throw new ProtocolEvidenceMissingException( + $"GetR result buffer is not a decodable NRBF stream: {ex.Message}"); + } + + if (root is not ClassRecord classRecord || !classRecord.TypeNameMatches(typeof(System.Data.DataTable))) + { + // TypeNameMatches needs the real type; fall back to the raw type name when the assembly + // identity differs from the local one. + if (root is not ClassRecord cr || !string.Equals(cr.TypeName.FullName, DataTableTypeName, StringComparison.Ordinal)) + { + throw new ProtocolEvidenceMissingException( + $"GetR result root record is not a {DataTableTypeName} (got '{(root as ClassRecord)?.TypeName.FullName ?? root.GetType().Name}')."); + } + classRecord = cr; + } + + if (!classRecord.HasMember(XmlSchemaMember) || !classRecord.HasMember(XmlDiffGramMember)) + { + throw new ProtocolEvidenceMissingException( + $"GetR DataTable record is missing the {XmlSchemaMember}/{XmlDiffGramMember} members (non-Xml SerializationFormat not supported)."); + } + + return (classRecord.GetString(XmlSchemaMember), classRecord.GetString(XmlDiffGramMember)); + } + + private static (string? tableName, List columns) ParseSchema(string schemaXml) + { + XDocument doc = XDocument.Parse(schemaXml); + XElement? schema = doc.Root; + List columns = []; + if (schema is null) + { + return (null, columns); + } + + // The dataset element advertises the main data table via msdata:MainDataTable. + XElement? dataSet = schema.Elements(Xsd + "element") + .FirstOrDefault(e => (string?)e.Attribute(MsData + "IsDataSet") == "true"); + string? mainTable = (string?)dataSet?.Attribute(MsData + "MainDataTable"); + + // The table element (named by MainDataTable, else the first non-dataset top-level element). + XElement? tableElement = schema.Elements(Xsd + "element") + .FirstOrDefault(e => mainTable is not null && (string?)e.Attribute("name") == mainTable) + ?? schema.Elements(Xsd + "element").FirstOrDefault(e => (string?)e.Attribute(MsData + "IsDataSet") != "true"); + + string? tableName = (string?)tableElement?.Attribute("name"); + + // Columns are the xs:element children of the table's complexType/sequence. + IEnumerable columnElements = tableElement + ?.Descendants(Xsd + "element") + .Where(e => e.Attribute("name") is not null) + ?? []; + foreach (XElement column in columnElements) + { + string name = (string)column.Attribute("name")!; + string type = (string?)column.Attribute("type") ?? InlineSimpleType(column) ?? "xs:string"; + columns.Add(new HistorianSqlColumn(name, type)); + } + + return (tableName, columns); + } + + private static string? InlineSimpleType(XElement column) + { + XElement? restriction = column.Element(Xsd + "simpleType")?.Element(Xsd + "restriction"); + return (string?)restriction?.Attribute("base"); + } + + private static IReadOnlyList> ParseDiffGram( + string? diffGramXml, + string? tableName, + List columns) + { + if (string.IsNullOrEmpty(diffGramXml) || columns.Count == 0) + { + return []; + } + + XDocument doc = XDocument.Parse(diffGramXml); + XElement? diffgram = doc.Root; + if (diffgram is null) + { + return []; + } + + // The dataset element is the diffgram's child that is not in the diffgr: namespace + // (diffgr:before / diffgr:errors are change-tracking sections we don't need for a SELECT). + XElement? dataSet = diffgram.Elements().FirstOrDefault(e => e.Name.Namespace != DiffGr); + if (dataSet is null) + { + return []; + } + + // Row elements are the dataset's children whose local name matches the table (or any child + // when the table name is unknown). + IEnumerable rowElements = dataSet.Elements() + .Where(e => tableName is null || e.Name.LocalName == tableName); + + List> rows = []; + foreach (XElement rowElement in rowElements) + { + object?[] values = new object?[columns.Count]; + for (int i = 0; i < columns.Count; i++) + { + HistorianSqlColumn column = columns[i]; + // ADO.NET maps a column either as a child element or as an attribute on the row. + XElement? cell = rowElement.Elements().FirstOrDefault(e => e.Name.LocalName == column.Name); + string? raw = cell?.Value + ?? rowElement.Attributes().FirstOrDefault(a => a.Name.LocalName == column.Name)?.Value; + values[i] = raw is null ? null : ConvertValue(raw, column.SchemaType); + } + rows.Add(values); + } + + return rows; + } + + private static object ConvertValue(string raw, string schemaType) + { + string type = schemaType.Contains(':') ? schemaType[(schemaType.IndexOf(':') + 1)..] : schemaType; + try + { + return type switch + { + "int" or "integer" => int.Parse(raw, CultureInfo.InvariantCulture), + "long" => long.Parse(raw, CultureInfo.InvariantCulture), + "short" => short.Parse(raw, CultureInfo.InvariantCulture), + "byte" => sbyte.Parse(raw, CultureInfo.InvariantCulture), + "unsignedByte" => byte.Parse(raw, CultureInfo.InvariantCulture), + "unsignedShort" => ushort.Parse(raw, CultureInfo.InvariantCulture), + "unsignedInt" => uint.Parse(raw, CultureInfo.InvariantCulture), + "unsignedLong" => ulong.Parse(raw, CultureInfo.InvariantCulture), + "double" => double.Parse(raw, CultureInfo.InvariantCulture), + "float" => float.Parse(raw, CultureInfo.InvariantCulture), + "decimal" => decimal.Parse(raw, CultureInfo.InvariantCulture), + "boolean" => ParseXsdBoolean(raw), + "dateTime" => DateTime.Parse(raw, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + "base64Binary" => Convert.FromBase64String(raw), + _ => raw, + }; + } + catch (Exception ex) when (ex is FormatException or OverflowException) + { + // Keep the raw string rather than throwing β€” the column type is informational and the + // server's textual rendering is always available. + return raw; + } + } + + private static bool ParseXsdBoolean(string raw) => + raw is "true" or "1" || (raw is not ("false" or "0") && bool.Parse(raw)); +} diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfSqlClient.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfSqlClient.cs new file mode 100644 index 0000000..59b99d8 --- /dev/null +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfSqlClient.cs @@ -0,0 +1,132 @@ +using System.ServiceModel; +using System.ServiceModel.Channels; +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Wcf.Contracts; + +namespace AVEVA.Historian.Client.Wcf; + +/// +/// Executes SQL commands (HCAL R1.1) over the 2020 WCF aa/Retr/ExeC + aa/Retr/GetR +/// ops. Both are string-handle ops reached with the Open2 storage-session GUID formatted UPPERCASE +/// (the same handle format that unlocked GETRP/GETHI; see wcf-string-handle-wall.md). The +/// record set comes back from GetR as an NRBF-serialized DataTable, parsed by +/// . +/// +internal static class HistorianWcfSqlClient +{ + // GetR is byte-stream-paged. A small record set returns the whole stream in the first call; + // larger ones chunk across calls accumulated here. The cap is a runaway guard. + private const int MaxPages = 4096; + + /// Diagnostic: the ExeC/GetR error from the last call (set only on server rejection). + public static string? LastError { get; private set; } + + public static Task ExecuteSqlCommandAsync( + HistorianClientOptions options, + string command, + HistorianSqlExecuteOption option, + CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(command); + return Task.Run(() => ExecuteSqlCommand(options, command, option), cancellationToken); + } + + private static HistorianSqlResult ExecuteSqlCommand( + HistorianClientOptions options, + string command, + HistorianSqlExecuteOption option) + { + Guid contextKey = Guid.NewGuid(); + var (histBinding, histEndpoint, retrBinding, retrEndpoint) = HistorianWcfBindingFactory.CreateBindingPair(options); + + LastError = null; + HistorianSqlResult? result = null; + HistorianWcfAuthChainHelper.OpenAuthenticatedConnection( + options, histBinding, histEndpoint, contextKey, CancellationToken.None, + additionalSetup: (_, context) => + result = RunSql(retrBinding, retrEndpoint, options, context.StorageSessionId, command, option)); + return result ?? new HistorianSqlResult([], [], 0); + } + + private static HistorianSqlResult RunSql( + Binding retrBinding, + EndpointAddress retrEndpoint, + HistorianClientOptions options, + Guid storageSessionId, + string command, + HistorianSqlExecuteOption option) + { + // ExeC/GetR take the storage-session GUID as a string handle, uppercase dash-no-braces. + string handle = storageSessionId.ToString("D").ToUpperInvariant(); + + ChannelFactory factory = new(retrBinding, retrEndpoint); + HistorianWcfClientCredentialsHelper.Configure(factory, options); + IRetrievalServiceContract3 channel = factory.CreateChannel(); + ICommunicationObject co = (ICommunicationObject)channel; + try + { + // Prime the Retrieval service version handshake (Retr.GetV). + channel.GetInterfaceVersion(out _); + + uint queryHandle = 0; + bool execOk = channel.ExecuteSqlCommand( + handle, command, (uint)option, ref queryHandle, + out int returnValue, out uint errSize, out byte[] errBuf); + if (!execOk) + { + LastError = $"ExeC returned false (retValue={returnValue}, errSize={errSize}, errLen={errBuf?.Length ?? 0})."; + return new HistorianSqlResult([], [], returnValue); + } + + using MemoryStream accumulated = new(); + uint sequence = 0; + for (int page = 0; page < MaxPages; page++) + { + bool getrOk = channel.GetRecordSetByteStream( + handle, queryHandle, ref sequence, + out uint resultSize, out byte[] resultBuffer, out uint gErrSize, out byte[] gErrBuf); + + // GetRResult is false even on a successful single-page read β€” the byte stream is in + // resultBuffer regardless, and false signals "this is the final page". So always + // consume the buffer first, then stop on a false result or an empty page. + if (resultBuffer is { Length: > 0 }) + { + accumulated.Write(resultBuffer, 0, resultBuffer.Length); + } + + if (!getrOk || resultBuffer is null || resultBuffer.Length == 0) + { + if (page == 0 && accumulated.Length == 0 && gErrBuf is { Length: > 0 }) + { + LastError = $"GetR error (resultSize={resultSize}, errSize={gErrSize}, errLen={gErrBuf.Length})."; + } + break; + } + } + + byte[] stream = accumulated.ToArray(); + DumpStreamIfRequested(stream); + return HistorianSqlResultProtocol.Parse(stream, returnValue); + } + finally + { + try { if (co.State == CommunicationState.Faulted) co.Abort(); else co.Close(); } catch { try { co.Abort(); } catch { } } + try { if (factory.State == CommunicationState.Faulted) factory.Abort(); else factory.Close(); } catch { try { factory.Abort(); } catch { } } + } + } + + /// + /// Reverse-engineering aid: when AVEVA_HISTORIAN_SQL_DUMP is set to a path, writes the + /// base64 of the reassembled GetR NRBF stream there (the clean contract-level byte[], free of + /// the MDAS transport chunk markers that corrupt a raw instrument-wcf capture). Used once to + /// produce the golden fixture for WcfSqlResultProtocolTests. + /// + private static void DumpStreamIfRequested(byte[] stream) + { + string? path = Environment.GetEnvironmentVariable("AVEVA_HISTORIAN_SQL_DUMP"); + if (!string.IsNullOrWhiteSpace(path) && stream.Length > 0) + { + try { File.WriteAllText(path, Convert.ToBase64String(stream)); } catch { /* diagnostic only */ } + } + } +} diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs index 1cd17ec..2b15da4 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs @@ -331,6 +331,65 @@ public sealed class HistorianClientIntegrationTests }); } + [Fact] + public async Task ExecuteSqlCommandAsync_AgainstLocalHistorian_ReturnsRecordSet() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe + }); + + // ExeC/GetR ride the storage-session GUID as an uppercase string handle. A constant SELECT + // returns a single int column; the DataTable is decoded from the NRBF stream (no BinaryFormatter). + AVEVA.Historian.Client.Models.HistorianSqlResult result = + await client.ExecuteSqlCommandAsync("SELECT 1 AS ProbeValue", cancellationToken: CancellationToken.None); + + AVEVA.Historian.Client.Models.HistorianSqlColumn column = Assert.Single(result.Columns); + Assert.Equal("ProbeValue", column.Name); + IReadOnlyList row = Assert.Single(result.Rows); + Assert.Equal(1, Assert.Single(row)); + } + + [Fact] + public async Task ExecuteSqlCommandAsync_AgainstLocalHistorian_MultiColumnMultiRow() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe + }); + + // Fully synthetic query (no server data): two columns (int + string), two rows, one NULL β€” + // exercises the schema/diffgram parser beyond the single-cell case. + AVEVA.Historian.Client.Models.HistorianSqlResult result = await client.ExecuteSqlCommandAsync( + "SELECT 10 AS Num, 'alpha' AS Word UNION ALL SELECT 20, NULL", + cancellationToken: CancellationToken.None); + + Assert.Equal(2, result.Columns.Count); + Assert.Equal("Num", result.Columns[0].Name); + Assert.Equal("Word", result.Columns[1].Name); + Assert.Equal(2, result.Rows.Count); + Assert.Equal(10, result.Rows[0][0]); + Assert.Equal("alpha", result.Rows[0][1]); + Assert.Equal(20, result.Rows[1][0]); + Assert.Null(result.Rows[1][1]); + } + [Fact] public async Task GetConnectionStatusAsync_AgainstLocalHistorian_ReportsConnectedToServer() { diff --git a/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs b/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs index b7454b5..14c0110 100644 --- a/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs @@ -66,6 +66,80 @@ public sealed class StringHandleProbeDiagnosticTests }); } + /// + /// R1.4 probe: GETHI on 2020 WCF is a named-value query (capture showed the native client + /// only ever asks for "HistorianVersion"; the 518-byte struct + EventStorageMode@514 is the + /// 2023R2 HCAL-native/gRPC model). This probes whether the 2020 server exposes any + /// storage-mode / extended info value through GETHI under other parameter names, so we can + /// decide honestly what GetHistorianInfoAsync can return over WCF. + /// + [Fact] + public void GETHI_CandidateInfoNames_AgainstLocalHistorian() + { + if (!ShouldRun(out string host)) return; + + HistorianClientOptions options = new() + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe + }; + + string[] candidates = + { + "HistorianVersion", "EventStorageMode", "EventStorageType", "StorageType", + "HistorianEventStorageMode", "EventStorage", "StorageMode", "HistorianInfo", + }; + + Dictionary results = new(); + ProbeOnStatusChannel(options, (channel, handle) => + { + foreach (string name in candidates) + { + using MemoryStream ms = new(); + using (BinaryWriter w = new(ms, Encoding.Unicode, leaveOpen: true)) + { + w.Write(new byte[] { 0x53, 0x67, 0x02, 0x00 }); + w.Write((uint)name.Length); + w.Write(Encoding.Unicode.GetBytes(name)); + } + byte[] req = ms.ToArray(); + byte[]? resp = null; + byte[]? err = null; + bool ok; + try + { + ok = channel.GetHistorianInfo(handle, req, out resp, out err); + } + catch (Exception ex) + { + _output.WriteLine($" {name,-26} EXCEPTION {ex.GetType().Name}: {ex.Message}"); + results[name] = (false, -1); + continue; + } + string respHex = resp is { Length: > 0 } + ? Convert.ToHexString(resp.AsSpan(0, Math.Min(48, resp.Length))) + : "(empty)"; + _output.WriteLine($" {name,-26} ok={ok} respLen={resp?.Length ?? 0} errLen={err?.Length ?? 0} resp={respHex}"); + results[name] = (ok, resp?.Length ?? 0); + } + }); + + // Locked finding (2026-06-20): on 2020 WCF, GETHI is a named-value query whose only + // working key is HistorianVersion. No storage-mode key is honored β€” EventStorageMode is + // not on the 2020 WCF wire (it is the 2023R2 HCAL-native/gRPC 518-byte-struct path). + Assert.True(results["HistorianVersion"].ok); + Assert.True(results["HistorianVersion"].respLen > 0); + foreach (string storageName in new[] + { + "EventStorageMode", "EventStorageType", "StorageType", + "HistorianEventStorageMode", "EventStorage", "StorageMode", + }) + { + Assert.False(results[storageName].ok, $"unexpected: GETHI honored '{storageName}'"); + } + } + [Fact] public void ExeC_WithUppercaseStorageGuid_AgainstLocalHistorian() { diff --git a/tests/AVEVA.Historian.Client.Tests/WcfSqlResultProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfSqlResultProtocolTests.cs new file mode 100644 index 0000000..614d03e --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/WcfSqlResultProtocolTests.cs @@ -0,0 +1,79 @@ +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Protocol; +using AVEVA.Historian.Client.Wcf; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Golden tests for the ExeC/GetR SQL result codec. The fixture is the real, clean GetR +/// pResultBuff captured from the live server for the benign query +/// SELECT 1 AS ProbeValue (no sensitive data) β€” an NRBF stream wrapping a +/// System.Data.DataTable serialized with SerializationFormat.Xml. It is the +/// contract-level byte[] the WCF channel reassembles (free of MDAS transport chunk markers). +/// +public sealed class WcfSqlResultProtocolTests +{ + // GetR pResultBuff for "SELECT 1 AS ProbeValue" (1232 bytes), base64. Captured via the + // AVEVA_HISTORIAN_SQL_DUMP diagnostic in HistorianWcfSqlClient. + private const string GetRStreamBase64 = + "AAEAAAD/////AQAAAAAAAAAMAgAAAE5TeXN0ZW0uRGF0YSwgVmVyc2lvbj00LjAuMC4wLCBDdWx0dXJlPW5ldXRyYWwsIFB1Ymxp" + + "Y0tleVRva2VuPWI3N2E1YzU2MTkzNGUwODkFAQAAABVTeXN0ZW0uRGF0YS5EYXRhVGFibGUDAAAAGURhdGFUYWJsZS5SZW1vdGlu" + + "Z1ZlcnNpb24JWG1sU2NoZW1hC1htbERpZmZHcmFtAwEBDlN5c3RlbS5WZXJzaW9uAgAAAAkDAAAABgQAAACJBTw/eG1sIHZlcnNp" + + "b249IjEuMCIgZW5jb2Rpbmc9InV0Zi0xNiI/Pg0KPHhzOnNjaGVtYSB4bWxucz0iIiB4bWxuczp4cz0iaHR0cDovL3d3dy53My5v" + + "cmcvMjAwMS9YTUxTY2hlbWEiIHhtbG5zOm1zZGF0YT0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTp4bWwtbXNkYXRhIj4NCiAg" + + "PHhzOmVsZW1lbnQgbmFtZT0iVGFibGUxIj4NCiAgICA8eHM6Y29tcGxleFR5cGU+DQogICAgICA8eHM6c2VxdWVuY2U+DQogICAg" + + "ICAgIDx4czplbGVtZW50IG5hbWU9IlByb2JlVmFsdWUiIG1zZGF0YTpSZWFkT25seT0idHJ1ZSIgdHlwZT0ieHM6aW50IiBtc2Rh" + + "dGE6dGFyZ2V0TmFtZXNwYWNlPSIiIC8+DQogICAgICA8L3hzOnNlcXVlbmNlPg0KICAgIDwveHM6Y29tcGxleFR5cGU+DQogIDwv" + + "eHM6ZWxlbWVudD4NCiAgPHhzOmVsZW1lbnQgbmFtZT0idG1wRGF0YVNldCIgbXNkYXRhOklzRGF0YVNldD0idHJ1ZSIgbXNkYXRh" + + "Ok1haW5EYXRhVGFibGU9IlRhYmxlMSIgbXNkYXRhOlVzZUN1cnJlbnRMb2NhbGU9InRydWUiPg0KICAgIDx4czpjb21wbGV4VHlw" + + "ZT4NCiAgICAgIDx4czpjaG9pY2UgbWluT2NjdXJzPSIwIiBtYXhPY2N1cnM9InVuYm91bmRlZCIgLz4NCiAgICA8L3hzOmNvbXBs" + + "ZXhUeXBlPg0KICA8L3hzOmVsZW1lbnQ+DQo8L3hzOnNjaGVtYT4GBQAAAJoCPGRpZmZncjpkaWZmZ3JhbSB4bWxuczptc2RhdGE9" + + "InVybjpzY2hlbWFzLW1pY3Jvc29mdC1jb206eG1sLW1zZGF0YSIgeG1sbnM6ZGlmZmdyPSJ1cm46c2NoZW1hcy1taWNyb3NvZnQt" + + "Y29tOnhtbC1kaWZmZ3JhbS12MSI+DQogIDx0bXBEYXRhU2V0Pg0KICAgIDxUYWJsZTEgZGlmZmdyOmlkPSJUYWJsZTExIiBtc2Rh" + + "dGE6cm93T3JkZXI9IjAiPg0KICAgICAgPFByb2JlVmFsdWU+MTwvUHJvYmVWYWx1ZT4NCiAgICA8L1RhYmxlMT4NCiAgPC90bXBE" + + "YXRhU2V0Pg0KPC9kaWZmZ3I6ZGlmZmdyYW0+BAMAAAAOU3lzdGVtLlZlcnNpb24EAAAABl9NYWpvcgZfTWlub3IGX0J1aWxkCV9S" + + "ZXZpc2lvbgAAAAAICAgIAgAAAAAAAAD//////////ws="; + + [Fact] + public void Parse_DataTableStream_ExtractsColumnAndTypedRow() + { + byte[] stream = Convert.FromBase64String(GetRStreamBase64); + + HistorianSqlResult result = HistorianSqlResultProtocol.Parse(stream, returnValue: 0); + + HistorianSqlColumn column = Assert.Single(result.Columns); + Assert.Equal("ProbeValue", column.Name); + Assert.Equal("xs:int", column.SchemaType); + + IReadOnlyList row = Assert.Single(result.Rows); + object? value = Assert.Single(row); + Assert.Equal(1, value); // typed to int per the xs:int schema + Assert.IsType(value); + Assert.Equal(0, result.ReturnValue); + } + + [Fact] + public void Parse_PreservesReturnValue() + { + byte[] stream = Convert.FromBase64String(GetRStreamBase64); + HistorianSqlResult result = HistorianSqlResultProtocol.Parse(stream, returnValue: 7); + Assert.Equal(7, result.ReturnValue); + } + + [Fact] + public void Parse_EmptyBuffer_ReturnsEmptyResult() + { + HistorianSqlResult result = HistorianSqlResultProtocol.Parse(ReadOnlyMemory.Empty, returnValue: 3); + Assert.Empty(result.Columns); + Assert.Empty(result.Rows); + Assert.Equal(3, result.ReturnValue); + } + + [Fact] + public void Parse_NonNrbfBuffer_Throws() + { + byte[] garbage = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]; + Assert.Throws( + () => HistorianSqlResultProtocol.Parse(garbage, returnValue: 0)); + } +} diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index 1273734..b2e54be 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -478,6 +478,33 @@ internal static class Program })); return 0; } + else if (openSuccess && status.ConnectedToServer && IsHistorianInfoScenario(scenario)) + { + // R1.4 capture: drive HistorianAccess.GetHistorianInfo(out HistorianInfo, out error) + // so instrument-wcf-{write,read}message can observe the WCF GETHI pRequestBuff that + // returns the full 518-byte HISTORIAN_INFO struct (version@0 + EventStorageMode@514), + // distinct from the named-value "HistorianVersion" request. Pure status read. + MethodInfo getInfoMethod = accessType.GetMethods() + .First(m => m.Name == "GetHistorianInfo" + && m.GetParameters().Length == 2 + && m.GetParameters()[0].ParameterType.IsByRef + && m.GetParameters()[1].ParameterType.IsByRef); + + object infoError = Activator.CreateInstance(errorType)!; + object?[] infoArgs = new object?[] { null, infoError }; + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-get-historian-info"); + bool infoOk = (bool)getInfoMethod.Invoke(access, infoArgs)!; + + Console.WriteLine(Serialize(new + { + Scenario = scenario, + GetHistorianInfoReturned = infoOk, + HistorianInfoType = getInfoMethod.GetParameters()[0].ParameterType.GetElementType()?.FullName, + HistorianInfo = infoArgs[0] is null ? null : SnapshotObject(infoArgs[0]!), + Error = SnapshotObject(infoArgs[1]!), + })); + return 0; + } else if (openSuccess && status.ConnectedToServer && IsEventSendScenario(scenario)) { // R2.1 capture: drive AddStreamedValue(HistorianEvent) and let instrument-wcf-* @@ -1770,6 +1797,19 @@ internal static class Program || scenario.Equals("runtime-parameter", StringComparison.OrdinalIgnoreCase); } + /// + /// Historian-info scenario (R1.4 capture): opens a normal authenticated process connection and + /// calls GetHistorianInfo(out HistorianInfo, out error) so instrument-wcf-{write,read}message + /// can observe the WCF GETHI pRequestBuff/pResponseBuff that returns the full + /// 518-byte HISTORIAN_INFO struct (version@0 + EventStorageMode@514). Pure status read. + /// + private static bool IsHistorianInfoScenario(string scenario) + { + return scenario.Equals("historian-info", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("hist-info", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("gethi", StringComparison.OrdinalIgnoreCase); + } + /// /// SQL-command scenario (R1.1 capture): opens a normal authenticated process connection and /// calls ExecuteSqlCommand (Retr.ExeC + Retr.GetR) so the string-handle SQL surface