From c1b1b3d23bb35edc74ab96dec56f5f1388af8c79 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 21 Jun 2026 11:26:21 -0400 Subject: [PATCH] R1.11 DelTep capture + R1.3/R1.4/R1.12/R1.13 bounded out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DelTep (extended-property delete) — wire format captured + serializer golden-proven, but live delete is server-blocked and NOT exposed publicly: - Captured the DelTep inBuff via a cross-session trick (harness add-tep gains --tep-skip-add + read-for-sync before --tep-delete; Capture-DeleteTagExtended Properties.ps1 / decode-del-tep-capture.py). Layout = same group framing as AddTEx but property-name-only (no 0x43 value) + 0x00 group trailer. - SerializeDeleteRequest + 4 golden tests pin the server-accepted buffer. - A decisive experiment shows SDK-added properties ARE deletable (the native client read-syncs and deletes one), so SDK-add is complete; the SDK's own DelTep is rejected by CHistStorage::DeleteTagExtendedProperties even with byte-identical inBuff, matching mode/handle, GetTgByNm+GetTepByNm prime, open channel, and 60s retries. Root cause: the native multiplexes services over one connection (per-connection working set); the SDK's per-service WCF channels don't reproduce it. Kept as documented-but-blocked internal orchestrator path; no public HistorianClient delete API. Bounded out with evidence (no code; docs + roadmap + probe): - R1.12 localized-property write — no op on 2020 (mirror of R1.6); no *LocalizedPropert*/TagLocalized* symbol in any current/*.dll. - R1.13 non-analog tag create — GATED; native AddTag rejects every non-analog type client-side (ValidationFailed, before any WCF op): SingleByteString, DoubleByteString, Int1 all fail, Float works. No Discrete type in the native enum, no TagType setter. No wire request to capture. - R1.3 timezone + R1.4 EventStorageMode — re-confirmed 2023R2/gRPC-only from the Runtime DB schema (no timezone param, no EventStorageMode anywhere) and a parameter-op probe (GetSystemParameter + GETRP return null/throw for every candidate; only HistorianVersion works). 238 unit tests pass; full solution builds with 0 warnings. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- docs/plans/hcal-roadmap.md | 10 +- .../wcf-add-tag-extended-properties.md | 73 ++++++++++- .../wcf-non-analog-tag-create.md | 67 ++++++++++ .../wcf-status-localhost.md | 28 ++++ .../wcf-tag-extended-properties.md | 12 ++ .../Capture-DeleteTagExtendedProperties.ps1 | 111 ++++++++++++++++ scripts/decode-del-tep-capture.py | 105 +++++++++++++++ src/AVEVA.Historian.Client/HistorianClient.cs | 9 ++ .../HistorianTagExtendedPropertyProtocol.cs | 53 ++++++++ .../Wcf/HistorianWcfTagWriteOrchestrator.cs | 120 +++++++++++++++++- .../HistorianClientIntegrationTests.cs | 6 + .../StringHandleProbeDiagnosticTests.cs | 39 ++++++ ...cfTagExtendedPropertyWriteProtocolTests.cs | 54 ++++++++ .../Program.cs | 77 ++++++++--- 14 files changed, 732 insertions(+), 32 deletions(-) create mode 100644 docs/reverse-engineering/wcf-non-analog-tag-create.md create mode 100644 scripts/Capture-DeleteTagExtendedProperties.ps1 create mode 100644 scripts/decode-del-tep-capture.py diff --git a/docs/plans/hcal-roadmap.md b/docs/plans/hcal-roadmap.md index f845e89..62b9f86 100644 --- a/docs/plans/hcal-roadmap.md +++ b/docs/plans/hcal-roadmap.md @@ -92,7 +92,7 @@ read/browse/status surface is Windows-free and the gRPC stack is the default pat |---|---|---|---| | R1.1 | `ExecuteSqlCommandAsync` | `Retrieval.ExecuteSqlCommand` (`ExeC`+`GetR`) | ✅ **REACHABLE (2026-06-20, live-probed).** The earlier "code 51 blocked" verdict was a handle-**format** bug — `ExeC` succeeds with the Open2 storage GUID sent **uppercase** (`ToString("D").ToUpperInvariant()`). Chain: `Retr.GetV` prime → `ExeC(handle, sqlString, option=0, ref queryHandle)` → `GetR(handle, queryHandle, ref sequence)` returns the result as a **BinaryFormatter-serialized .NET DataTable**. Proven by `StringHandleProbeDiagnosticTests` + `scripts/Capture-ExecSql.ps1`. **Public API not yet shipped** — needs a `GetR` continuation loop + a custom BinaryFormatter-stream parser (BinaryFormatter is removed from .NET 10, so a DataTable can't just be deserialized). | | ~~R1.2~~ | ~~`GetRuntimeParameterAsync`~~ | `Status.GetRuntimeParameter` (`aa/Stat/GETRP`) | ✅ **DONE (2026-06-20), live-verified.** Captured (`scripts/Capture-RuntimeParam.ps1`): GETRP is a **`string`-handle** op (GETHI's shape), but reachable from the managed client using the Open2 storage-session GUID as an **uppercase** string handle (`ToString("D").ToUpperInvariant()`). Returns `HistorianVersion` = `20,0,000,000` live. pRequestBuff = `54 67 01 00` + uint nameCount + per-name(uint charCount + UTF-16); pResponseBuff = version + uint resultCount + CRetVariant(`0x43` VT_BSTR + uint16 len + uint16 charCount + UTF-16). Single string-valued param only (multi-name framing inferred, not captured). Shipped: `HistorianClient.GetRuntimeParameterAsync(name)`; golden `WcfRuntimeParameterProtocolTests`. **Note:** GETRP punching through the string-handle wall with the uppercase storage GUID is a strong lead that GETHI/ExeC may be a handle-*format* issue — see `wcf-string-handle-wall.md` §Update. | -| ~~R1.3~~ | ~~`GetServerTimeZoneAsync`~~ | `Status.GetSystemTimeZoneName` | ⚠ **gRPC/2023R2-only.** Verified 2026-06-20: over **2020 WCF** this op is a stub (rc=0, empty value) in the `GetServerTime` family — not shippable here. Build+verify only against a live 2023 R2 server. See `docs/reverse-engineering/wcf-status-localhost.md`. | +| ~~R1.3~~ | ~~`GetServerTimeZoneAsync`~~ | `Status.GetSystemTimeZoneName` | ⛔ **gRPC/2023R2-only — re-confirmed 2026-06-21 from 3 angles.** (1) native `GetSystemTimeZoneName` is a stub (rc=0, empty) in the `GetServerTime` family; (2) Runtime DB has no timezone `SystemParameter` — the zone exists only as per-block `HistoryBlock.TimeZoneOffset`/`wwTimeZone` (DST-specific, SQL-only) + a `TimeZone` lookup table, `StorageShard.TimeZoneId`=NULL; (3) `GetSystemParameter` + `GETRP` return null/throw for every timezone candidate (only `HistorianVersion` works). The sole 2020 route is a SQL read via `ExecuteSqlCommand` (R1.1) — DST-specific, different mechanism. Build the real op only against a live 2023 R2 server. See `docs/reverse-engineering/wcf-status-localhost.md`. | > ✅ **String-handle "wall" RESOLVED (2026-06-20) — it was a handle-FORMAT bug.** R1.4/R1.5/R1.6 > (and R1.1) take a **`string` GUID handle**; the earlier "code 1/51 blocked" verdict came from @@ -110,7 +110,7 @@ read/browse/status surface is Windows-free and the gRPC stack is the default pat ### 1b. Bounded (decode one `bytes` payload; S–M each) | ID | Capability | gRPC op | Payload to decode | Depends | |---|---|---|---|---| -| R1.4 | `GetHistorianInfoAsync` | `Status.GetHistorianInfo` (`GETHI`) | ✅ **REACHABLE (2026-06-20, live-probed)** via the uppercase storage GUID — `GETHI` returns data (`StringHandleProbeDiagnosticTests`). The version-keyed request returns `uint charCount + UTF-16`; the full info struct (incl. `EventStorageMode`@514) needs its own request capture. **Public API not yet shipped.** | uppercase string handle | +| ~~R1.4~~ | `GetHistorianInfoAsync` | `Status.GetHistorianInfo` (`GETHI`) | ⛔ **BOUNDED OUT on 2020 (re-confirmed 2026-06-21).** GETHI is a **named-value** query reachable via the uppercase storage GUID, but **only `HistorianVersion` resolves** — the full 518-byte `HISTORIAN_INFO` struct (incl. `EventStorageMode`@514) is the 2023R2 HCAL-native/gRPC model. `EventStorageMode` has **no 2020 representation at all**: not a `SystemParameter` (only `EventStorageDuration`/`EventStorageLogPath`), not a DB column, and `GETRP`/`GetSystemParameter` return null/throw for it. The only 2020-reachable field (version) is already shipped via `GetSystemParameterAsync`/`GetRuntimeParameterAsync`, so a struct API would be hollow + misleading. Build the real op only against a live 2023 R2 server. See `docs/reverse-engineering/wcf-status-localhost.md`. | uppercase string handle | | ~~R1.5~~ | Extended-property **read** | `Retrieval.GetTagExtendedPropertiesFromName` (`GetTepByNm`) | ✅ **DONE (2026-06-20), live-verified.** `GetTagExtendedPropertiesAsync(tag)` → name/value pairs. String-handle op via the uppercase storage GUID; name-based path (`GetTagExtendedPropertiesByName`, not the QTB-gated TagQuery path). Request `tagNames` = `uint count` + per-name(`uint charCount`+UTF-16); response = `uint tagCount` + per-tag(marker + compact-ASCII name + `uint propCount` + per-prop(marker + compact-ASCII name + `0x43` VT_BSTR value) + trailer). Sequence-paged. Shipped: `HistorianTagExtendedPropertyProtocol`, golden `WcfTagExtendedPropertyProtocolTests`, gated live test. See `docs/reverse-engineering/wcf-tag-extended-properties.md`. | uppercase string handle | | ~~R1.6~~ | Localized-property **read** | (no op) | ⛔ **No distinct op on 2020 — collapses into R1.5.** There is no `GetTagLocalizedPropertiesFromName`/`GetTlpByNm` or `GetTagLocalizedPropertiesByName` in `current/aahClientManaged.dll`; the only "localized" surfaces are error-message/UI-text localization. Extended properties (R1.5) are the user-defined tag-property read surface. Closed, not throwing. | — | | ~~R1.7~~ | Event **filters** | filter bytes in `Retrieval.StartEventQuery` | ✅ **DONE (2026-06-20), live-honored.** `ReadEventsAsync(start, end, HistorianEventFilter)`. The filter rides `StartEventQuery`'s `pRequestBuff` (captured via `EventQuery.AddEventFilter` + instrument-wcf-writemessage; Equal vs Contains diffed to isolate the op). Filter block: `ushort 0 + uint filterCount + uint condCount + uint nameLen + name(UTF-16) + uint 1 + ushort op + uint 1 + value(0x09-len-0x00 compact-ASCII) + byte 0`. **REAL, not inert** (a non-matching predicate returns 0 events; matching returns the subset). Single string-valued predicate only; multi-filter (OR) / multi-condition (AND via `AddEventFilterCondition`) framing not yet fully captured. See `HistorianEventFilter`, golden `WcfEventQueryProtocolTests`. | — | @@ -121,9 +121,9 @@ read/browse/status surface is Windows-free and the gRPC stack is the default pat | ID | Capability | gRPC op | Payload | Notes | |---|---|---|---|---| | R1.10 | `RenameTagsAsync` | History rename op | rename request buffer | `AllowRenameTags` already probed | -| ~~R1.11~~ | Extended-property **write** | `History.AddTagExtendedProperties` (AddTEx) | ✅ **Add DONE (2026-06-21), live-verified.** `AddTagExtendedPropertiesAsync`/`AddTagExtendedPropertyAsync` (write mode, uppercase handle). inBuff = exact inverse of the R1.5 read framing (`uint32 groupCount + 0x01 + compact-ASCII tag + uint32 propCount + per prop[0x02 + compact-ASCII name + 0x43 VT_BSTR value] + 0x01 trailer + 0x00 terminator`); the trailing `0x00` is required or the server throws. Golden `WcfTagExtendedPropertyWriteProtocolTests` + gated live write/read-back test. **Delete (DelTep) deferred** — native client-side sync gate (err 229) blocks capturing its inBuff. See `docs/reverse-engineering/wcf-add-tag-extended-properties.md`. | -| R1.12 | Localized-property **write** | `History.AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` | localized serialize | | -| R1.13 | Non-analog tag create (string/discrete) | `History.EnsureTags` | distinct CTagMetadata variant | ⚠ native AddTag rejected some types — confirm server path first; may be GATED | +| ~~R1.11~~ | Extended-property **write** | `History.AddTagExtendedProperties` (AddTEx) | ✅ **Add DONE (2026-06-21), live-verified.** `AddTagExtendedPropertiesAsync`/`AddTagExtendedPropertyAsync` (write mode, uppercase handle). inBuff = exact inverse of the R1.5 read framing (`uint32 groupCount + 0x01 + compact-ASCII tag + uint32 propCount + per prop[0x02 + compact-ASCII name + 0x43 VT_BSTR value] + 0x01 trailer + 0x00 terminator`); the trailing `0x00` is required or the server throws. Golden `WcfTagExtendedPropertyWriteProtocolTests` + gated live write/read-back test. **Delete (DelTep): wire format CAPTURED + serializer golden-proven (2026-06-21), but live delete is server-blocked and NOT shipped.** Captured via a two-session trick (add in Run A → fresh-session read-sync → delete in Run B, past the native err-229 client gate); inBuff = same group framing as Add but property-name-only and a `0x00` group trailer. A decisive experiment shows SDK-added properties ARE deletable (the native client deletes one), so SDK-add is complete; the SDK's own DelTep is rejected (`SErrorException` in `CHistStorage::DeleteTagExtendedProperties`) despite matching mode/handle/inBuff + GetTgByNm/GetTepByNm prime + open channel + 60s retries. Root cause: the native multiplexes services over ONE connection (per-connection working set), which the SDK's per-service WCF channels don't reproduce — needs transport-level multiplexing. See `docs/reverse-engineering/wcf-add-tag-extended-properties.md` §Delete. | +| ~~R1.12~~ | Localized-property **write** | (no op) | ⛔ **No distinct op on 2020 — closed (mirror of R1.6).** A symbol sweep of `current/*.dll` finds no `AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` / any `*LocalizedPropert*` / `TagLocalized*`; only UI/error-text localization (`GetLocalizedText`/`GetLocalizedMessage`/`LocalizedResourcesDir`). Localized properties are a 2023 R2/gRPC concept. Closed, not throwing. See `docs/reverse-engineering/wcf-tag-extended-properties.md` §R1.12. | 2026-06-21 | +| ~~R1.13~~ | Non-analog tag create (string/discrete) | `History.EnsureTags` | distinct CTagMetadata variant | ⛔ **GATED — bounded out (2026-06-21, live-probed).** Native `AddTag` rejects every non-analog type **client-side** (`ErrorCode=ValidationFailed` / "Transaction validation failed", before any WCF op): SingleByteString, DoubleByteString, **and Int1** all fail; Float (control) succeeds. The native `HistorianDataType` enum has **no Discrete/Boolean** and no Int8/UInt8 (SDK-only extensions); `HistorianTag` has **no TagType setter** (type is data-type-derived). So no non-analog wire request is ever emitted → nothing to capture/implement. String/discrete create goes via a different subsystem (config editor / SQL), not this client's AddTag. `EnsureTagAsync` stays analog-only. See `docs/reverse-engineering/wcf-non-analog-tag-create.md`. | **Acceptance:** read + browse + metadata + system/status + property R/W + summaries + event-filtered reads + rename all live-verified over gRPC. diff --git a/docs/reverse-engineering/wcf-add-tag-extended-properties.md b/docs/reverse-engineering/wcf-add-tag-extended-properties.md index 2c5c076..27c5ad2 100644 --- a/docs/reverse-engineering/wcf-add-tag-extended-properties.md +++ b/docs/reverse-engineering/wcf-add-tag-extended-properties.md @@ -47,14 +47,73 @@ instrument capture mangles the final byte with MDAS chunk markers, so the golden **clean** byte[] the SDK handed the channel (dumped via `AVEVA_HISTORIAN_TEP_DUMP`) — the exact buffer the live server accepted. -## Delete (DelTep) — deferred +## Delete (DelTep) — wire format captured + serializer proven; live delete server-blocked -`DeleteTagExtendedProperties` (`DelTep`) is **not shipped yet**. The native -`DeleteTagExtendedPropertiesByName(tag, propertyNames, deleteFromServer, out err)` performs a -**client-side sync check** and returns error **229 ("Tag extended property not synchronized with -server")** when deleting a just-added property — so the `DelTep` request never reached the wire and its -inBuff could not be captured. Capturing it needs a property that is already server-synchronized (add it -in one session, then delete in a later one). Left for a follow-up rather than shipping a guessed buffer. +**Status (2026-06-21): the `DelTep` wire format is captured and decoded, the serializer is +golden-verified against a server-accepted buffer, and SDK-added properties are confirmed deletable — +but the SDK's own delete is rejected server-side and is therefore NOT exposed publicly.** This is a +much deeper result than the earlier "couldn't capture the inBuff" deferral. + +### Capturing it: the cross-session trick + +The native `DeleteTagExtendedPropertiesByName(tag, propertyNames, deleteFromServer, out err)` performs +a **client-side sync check** and returns error **229 ("Tag extended property not synchronized with +server")** when deleting a *just-added* property — so a same-session add→delete never reaches the wire. +`AddTEx` success does **not** mark the local cache entry as server-synchronized; only a *server fetch* +(`GetTepByNm`) does. So the capture (`scripts/Capture-DeleteTagExtendedProperties.ps1`) runs two +separate harness processes against one instrumented DLL: + +- **Run A**: `add-tep` creates the sandbox tag and adds the property (now server-synced). +- **Run B**: a fresh process opens a new connection, fetches the property + (`GetTagExtendedPropertiesByName`, which seeds the local cache as synced), then deletes it — so + `DeleteTagExtendedPropertiesByName` passes the client gate and `DelTep` reaches the wire. + +### The inBuff — same group framing as Add, names only + +``` +uint32 groupCount (= 1) +byte 0x01 (group marker) +0x09 + uint16 byteLen + ASCII tagName +uint32 propertyCount +repeated propertyCount times: + byte 0x02 (property marker) + 0x09 + uint16 byteLen + ASCII propertyName ← NO value variant +byte 0x00 (group trailer) ← 0x00 for delete, vs 0x01 for add +byte 0x00 (buffer terminator) +``` + +The native `deleteFromServer` argument is **not** in the buffer — it is the client-side flag that +decides whether the wire op fires at all (`true` ⇒ a `DelTep` call). `HistorianTagExtendedProperty +Protocol.SerializeDeleteRequest` produces this exactly; `WcfTagExtendedPropertyWriteProtocolTests` +pins the server-accepted bytes. + +### Why the SDK delete is server-blocked + +The SDK's `DelTep` is rejected by the server with `SErrorException` in +`aahClientAccessPoint::CHistStorage::DeleteTagExtendedProperties`, even though: + +- the **inBuff is byte-identical** to the server-accepted native capture (golden-verified); +- the **Open2 connection mode matches** the native (`0x401`, confirmed from the capture at offset + 0x4a4); +- the **handles match** (uppercase storage-session GUID for `DelTep`/`GetTepByNm`, uint client handle + for `GetTgByNm`); +- the SDK first **primes the session** with `GetTgByNm` (tag identity, returns 140 bytes of tag info) + and `GetTepByNm` (returns the property), keeping the Retrieval prime channel **open across** the + `DelTep` call; +- retried with backoff for **60 s** (ruling out a storage-tier sync delay). + +A decisive experiment localizes the gap: an **SDK-added** property *is* deletable — the native client +read-syncs and deletes it (`Success:true`). So the SDK's **add is complete**; only the SDK's **delete +session** is the problem. The native client multiplexes Hist/Retr/Stat/Trx over **one connection** +under a single `HistorianAccess` session, so its `GetTepByNm` populates a **per-connection working set** +that the same-connection `DelTep` consults. The SDK uses **separate WCF channels per service** (the +proven read pattern), so the borrowed-GUID Retrieval prime doesn't satisfy that server-side check. +Reproducing it requires transport-level connection multiplexing — a substantial change beyond this op. + +The investigated-but-blocked orchestration is kept (internal +`HistorianWcfTagWriteOrchestrator.DeleteTagExtendedPropertiesAsync`, the `PrimeThenDelete` helper) for +follow-up, but `HistorianClient` deliberately exposes **no** public delete to avoid a silently-failing +write API. Capture/decode tooling: `scripts/Capture-DeleteTagExtendedProperties.ps1` + +`scripts/decode-del-tep-capture.py`, harness `add-tep` scenario with `--tep-skip-add` / `--tep-delete`. ## Shipped surface diff --git a/docs/reverse-engineering/wcf-non-analog-tag-create.md b/docs/reverse-engineering/wcf-non-analog-tag-create.md new file mode 100644 index 0000000..5565ac2 --- /dev/null +++ b/docs/reverse-engineering/wcf-non-analog-tag-create.md @@ -0,0 +1,67 @@ +# Non-analog tag create over 2020 WCF — GATED (HCAL R1.13) + +**Status: ⛔ bounded out (2026-06-21). No non-analog (string / discrete / wide-integer) tag-create +path is reachable on 2020 — the native managed client rejects every non-analog type *client-side*, +before any WCF op, so there is no wire format to capture and nothing to implement against.** +`EnsureTagAsync` stays analog-only (Float, Double, Int2, UInt2, Int4, UInt4); unsupported types throw +`ProtocolEvidenceMissingException` from `HistorianTagWriteProtocol.GetAnalogDataTypeCode`. + +## What R1.13 asked for + +Create string / discrete (non-analog) tags via `History.EnsureTags`, with a distinct `CTagMetadata` +variant. The roadmap flagged it "⚠ native AddTag rejected some types — confirm server path first; +may be GATED." + +## Findings (live-probed against the local 2020 Historian) + +1. **No discrete/boolean data type exists.** The native `ArchestrA.HistorianDataType` enum + (`current/aahClientManaged.dll`, dumped via `enum-dump`) has exactly 12 members: `Int1, Int2, + UInt2, Int4, UInt4, Float, Double, SingleByteString, DoubleByteString, Event, Structure`. There is + no `Discrete`/`Boolean`, and no `Int8`/`UInt8`/`UInt1`/`Guid`/`FileTime` (those are SDK-only + extensions in `Models/HistorianDataType`, recovered from the C++ `CDataType` predicate IL — they + are not settable on the managed `HistorianTag`). + +2. **Tag type is data-type-derived, not separately settable.** `ArchestrA.HistorianTag` + (`--dump-type-members`) has **no** `TagType` property — only `TagDataType`. It does carry the + discrete/string-shaped fields (`MessageOn`/`MessageOff`, `RolloverValue`, dead-band/interpolation), + and the type exposes `ValidateAnalog*`, `ValidateDiscreteGeneralProperties`, and + `ValidateDiscreteAndStringStorageProperties` — but the analog-vs-discrete-vs-string decision is made + internally from the data type, with no way to request "discrete." + +3. **Native AddTag rejects every non-analog type client-side.** Driving the native + `HistorianAccess.AddTag(HistorianTag, …)` (harness `write` scenario, `--write-data-type`) against + the live server: + + | Data type | AddTag.Success | ErrorCode | ErrorType | + |---|---|---|---| + | SingleByteString | **false** | ValidationFailed | CustomError | + | DoubleByteString | **false** | ValidationFailed | CustomError | + | Int1 | **false** | ValidationFailed | CustomError | + | Int8 / UInt8 | n/a | *not in the native enum* | — | + | Float (control) | true | Success | — | + + The error — `ErrorType=CustomError`, `ErrorCode=ValidationFailed`, `ErrorDescription="Transaction + validation failed"` — is raised by the client's own `Validate*` chain **before any WCF message is + sent** (the wrapper even auto-populates discrete defaults `MessageOn=ON`/`MessageOff=OFF`, then + fails validation). So the native client never emits a non-analog `EnsT2`/`AddTag` request. + +## Why it's not deliverable here + +Because the native client refuses non-analog types client-side, **no wire request exists to +reverse-engineer** — there is no captured `CTagMetadata` variant for string or discrete tags, and the +SDK does not guess wire bytes. The rejection is not string-specific: `Int1` (a non-string integer +outside the analog set `{Float, Double, Int2, UInt2, Int4, UInt4}`) fails identically, so the boundary +is "the analog set" rather than "strings only." Creating string/discrete tags on 2020 evidently goes +through a different subsystem (e.g. the configuration editor / SQL config path), not this client's +`AddTag`. R1.13 is closed as GATED, consistent with the mission note that these types "fail at native +AddTag — likely require a different path and are intentionally not supported." + +## Probe commands (read-only / sandbox-guarded) + +``` +dotnet run --project tools\AVEVA.Historian.ReverseEngineering -- enum-dump current\aahClientManaged.dll HistorianDataType +dotnet run --project tools\AVEVA.Historian.ReverseEngineering -- dnlib-method current\aahClientManaged.dll HistorianTag.ValidateDataType +# native AddTag probe (sandbox tag must start with RetestSdkWrite; --write-skip-add-value avoids the blocked value path): +dotnet run --project tools\AVEVA.Historian.NativeTraceHarness -- --scenario write --server-name localhost --tcp-port 32568 \ + --write-sandbox-tag RetestSdkWriteNADoubleByteString --write-data-type DoubleByteString --write-skip-add-value +``` diff --git a/docs/reverse-engineering/wcf-status-localhost.md b/docs/reverse-engineering/wcf-status-localhost.md index 03de8ed..4fbb24a 100644 --- a/docs/reverse-engineering/wcf-status-localhost.md +++ b/docs/reverse-engineering/wcf-status-localhost.md @@ -71,3 +71,31 @@ Findings: Shipped as `HistorianClient.GetRuntimeParameterAsync(name)`. See `HistorianRuntimeParameterProtocol`, golden `WcfRuntimeParameterProtocolTests`, and the handle-format lead in `wcf-string-handle-wall.md` §Update (retry GETHI/ExeC uppercased). + +## R1.3 timezone + R1.4 EventStorageMode — re-confirmed bounded out (2026-06-21) + +Both were already classified 2023R2/gRPC-only; re-verified from two *fresh* angles that corroborate it +more strongly than the original op-level probes: + +- **Runtime DB schema** (`Runtime.dbo`, the server's own source of truth): the `SystemParameter` table + has **no** timezone parameter and **no `EventStorageMode`** (only `EventStorageDuration` / + `EventStorageLogPath`). The server timezone exists only as **per-block storage artifacts** + (`HistoryBlock.TimeZoneOffset` = e.g. 240 min, `wwTimeZone` = e.g. "Eastern Daylight Time") and a + `TimeZone` reference/lookup table; `StorageShard.TimeZoneId` is NULL. So the timezone is a + DST-specific, SQL-only, OS-derived value, not a clean server-config field exposed by any op. +- **Parameter-op probe** (`StringHandleProbeDiagnosticTests.TimezoneAndStorageMode_ParameterProbe`): + `GetSystemParameter` and `GetRuntimeParameter` (GETRP) were asked for every timezone candidate + (`TimeZone`/`ServerTimeZone`/`SystemTimeZone`/`TimeZoneName`/`SystemTimeZoneName`/`TimeStampRule`/ + `ServerTime`) and every storage-mode candidate (`EventStorageMode`/`StorageMode`/`EventStorage`/ + `EventStorageDuration`). **All returned null (GetSystemParameter) or threw + `ProtocolEvidenceMissingException` (GETRP — non-string/empty response)**; only the `HistorianVersion` + control returned a value (`20,0,000,000`). Note: `TimeStampRule`/`EventStorageDuration` *do* exist in + the `SystemParameter` table yet `GetSystemParameterAsync` returns null for them — the shipped op only + surfaces a whitelisted subset (a possible future widening, unrelated to R1.3/R1.4). + +Conclusion: **R1.3 `GetServerTimeZoneAsync` and R1.4 `GetHistorianInfoAsync` (EventStorageMode) are not +deliverable as server ops on 2020.** The only 2020 route to the timezone is a SQL read of +`HistoryBlock`/`TimeZone` via `ExecuteSqlCommand` (R1.1) — a DST-specific value over a different +mechanism than the roadmap's `Status.GetSystemTimeZoneName`. `EventStorageMode` has no 2020 +representation at all (it is a 2023 R2 event-storage-architecture field). Deliver both only against a +live 2023 R2 gRPC server. diff --git a/docs/reverse-engineering/wcf-tag-extended-properties.md b/docs/reverse-engineering/wcf-tag-extended-properties.md index e0d4816..785fb54 100644 --- a/docs/reverse-engineering/wcf-tag-extended-properties.md +++ b/docs/reverse-engineering/wcf-tag-extended-properties.md @@ -94,6 +94,18 @@ UI-text localization), not tag properties. So R1.6 **collapses into R1.5**: exte (`GetTepByNm`) are the user-defined tag-property read surface on 2020. R1.6 is closed as "no separate op," not left throwing. +## R1.12 (localized-property write) — no distinct op on 2020 (mirror of R1.6) + +Symmetric to R1.6 on the write side: there is **no** `AddTagLocalizedProperties` / +`DeleteTagLocalizedProperties` (or any `*LocalizedPropert*` / `TagLocalized*`) symbol in **any** +`current/*.dll`. A full symbol sweep of the shipped client DLLs surfaces only `GetLocalizedText`, +`GetLocalizedMessage`, and `LocalizedResourcesDir` — all UI/error-message-text localization, not tag +data. (The sweep does find the real write op `AddTagExtendedProperties` and the whole `AddTag*` +family, so the absence of a localized op is a true negative, not a grep miss.) R1.12 is therefore +**closed as "no separate op"** — the same conclusion as R1.6's read side. Extended-property write +(R1.11 `AddTEx`) is the user-defined tag-property write surface on 2020; localized properties are a +2023 R2 / gRPC-only concept. Not left throwing. + ## Shipped surface - `HistorianClient.GetTagExtendedPropertiesAsync(tag)` → `IReadOnlyList` diff --git a/scripts/Capture-DeleteTagExtendedProperties.ps1 b/scripts/Capture-DeleteTagExtendedProperties.ps1 new file mode 100644 index 0000000..d73ea60 --- /dev/null +++ b/scripts/Capture-DeleteTagExtendedProperties.ps1 @@ -0,0 +1,111 @@ +<# +.SYNOPSIS + Captures the native AVEVA client's DeleteTagExtendedProperties (DelTep) wire traffic (HCAL R1.11 + delete half) using the CROSS-SESSION trick so the delete passes the client-side sync gate. + +.DESCRIPTION + DeleteTagExtendedPropertiesByName does a CLIENT-SIDE sync check and returns err 229 ("Tag extended + property not synchronized with server") for any property the local cache doesn't see as + server-synchronized — so a just-added property can't be deleted in the same session and its DelTep + inBuff never reaches the wire. This script captures it in two SEPARATE harness processes (= two + sessions) against one instrumented aahClientManaged.dll: + + Run A: add-tep (create sandbox tag + AddTagExtendedProperties) -> property now server-synced + Run B: add-tep --tep-skip-create --tep-skip-add --tep-delete -> fresh connection: fetch the + property from the server (seeds the local cache as SYNCED), then + DeleteTagExtendedPropertiesByName, which now reaches the wire as DelTep. + + The capture file is cleared BETWEEN the two runs so it contains only Run B (the GetTepByNm + read-for-sync + the DelTep delete). Decode with scripts/decode-del-tep-capture.py. + + SAFETY: sandbox-guarded — the tag MUST start with 'RetestSdkWrite'. The run leaves the sandbox tag + in place (property removed); delete the tag afterwards with the supported aaDeleteTag proc. + +.NOTES + Artifacts are diagnostic and gitignored. Sanitize before copying into docs/. +#> +[CmdletBinding()] +param( + [string]$ServerName = "localhost", + [int]$TcpPort = 32568, + [string]$TepTag = "RetestSdkWriteDelTepSdk", + [string]$PropName = "SdkDelProp", + [string]$PropValue = "SdkDelValue", + [string]$Configuration = "Debug" +) + +$ErrorActionPreference = "Stop" +$repoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $repoRoot +if (-not $TepTag.StartsWith("RetestSdkWrite")) { throw "-TepTag must start with 'RetestSdkWrite' (sandbox guard)." } + +$reProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseEngineering\AVEVA.Historian.ReverseEngineering.csproj" +$harnessProj = Join-Path $repoRoot "tools\AVEVA.Historian.NativeTraceHarness\AVEVA.Historian.NativeTraceHarness.csproj" +$instrProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\AVEVA.Historian.ReverseInstrumentation.csproj" + +$captureDir = Join-Path $repoRoot "artifacts\reverse-engineering\instrumented-wcf-del-tep" +$currentCopy = Join-Path $captureDir "current-copy" +$instrDll = Join-Path $captureDir "aahClientManaged.dll" +$capturePath = Join-Path $captureDir "del-tep-capture-latest.ndjson" + +Write-Host "== Building tooling ($Configuration) ==" -ForegroundColor Cyan +dotnet build $reProj -c $Configuration --nologo -v q | Out-Null +dotnet build $instrProj -c $Configuration --nologo -v q | Out-Null +dotnet build $harnessProj -c $Configuration --nologo -v q | Out-Null + +$instrSourceDll = Get-ChildItem -Recurse (Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\bin\$Configuration") ` + -Filter "AVEVA.Historian.ReverseInstrumentation.dll" | Select-Object -First 1 -ExpandProperty FullName +if (-not $instrSourceDll) { throw "ReverseInstrumentation.dll not found under bin\$Configuration." } + +Write-Host "== Instrumenting WriteMessage + ReadMessage ==" -ForegroundColor Cyan +New-Item -ItemType Directory -Force -Path $captureDir | Out-Null +$writeOnly = Join-Path $captureDir "aahClientManaged.write.dll" +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-writemessage (Join-Path $repoRoot "current\aahClientManaged.dll") $writeOnly | Out-Null +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-readmessage $writeOnly $instrDll | Out-Null + +Write-Host "== Staging current-copy ==" -ForegroundColor Cyan +robocopy (Join-Path $repoRoot "current") $currentCopy /MIR /NJH /NJS /NDL /NP /NC /NS | Out-Null +Copy-Item -Force $instrDll (Join-Path $currentCopy "aahClientManaged.dll") +Copy-Item -Force $instrSourceDll (Join-Path $currentCopy "AVEVA.Historian.ReverseInstrumentation.dll") + +$harnessDll = Join-Path $currentCopy "aahClientManaged.dll" +$env:AVEVA_HISTORIAN_RE_CAPTURE = $capturePath + +function Invoke-Harness([string[]]$extraArgs, [string]$label) { + Write-Host "== $label ==" -ForegroundColor Green + $harnessArgs = @( + "--scenario", "add-tep", + "--server-name", $ServerName, + "--tcp-port", "$TcpPort", + "--tep-tag", $TepTag, + "--tep-name", $PropName, + "--tep-value", $PropValue, + "--current-dir", $currentCopy, + "--managed-dll-path", $harnessDll + ) + $extraArgs + $json = $null + try { + $prevEap = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $json = & dotnet run --no-build -c $Configuration --project $harnessProj -- @harnessArgs 2>&1 + } finally { $ErrorActionPreference = $prevEap } + $json | Select-Object -Last 20 +} + +# Run A: create the sandbox tag + add the property (server-synced afterwards). +Invoke-Harness @() "Run A: create + AddTagExtendedProperties ($TepTag : $PropName=$PropValue)" + +# Clear the capture so the file contains only Run B (read-for-sync + DelTep). +if (Test-Path $capturePath) { Remove-Item -Force $capturePath } + +# Run B: FRESH session — fetch (sync the local cache) then DeleteTagExtendedPropertiesByName. +Invoke-Harness @("--tep-skip-create", "--tep-skip-add", "--tep-delete") "Run B: fresh session -> read-for-sync -> DelTep" + +Remove-Item Env:\AVEVA_HISTORIAN_RE_CAPTURE -ErrorAction SilentlyContinue + +$recCount = if (Test-Path $capturePath) { (Get-Content $capturePath | Where-Object { $_.Trim() }).Count } else { 0 } +Write-Host "`n== Capture summary (Run B only) ==" -ForegroundColor Cyan +Write-Host " -> $recCount records -> $capturePath" +Write-Host "`nDecode with: python scripts\decode-del-tep-capture.py" -ForegroundColor Cyan diff --git a/scripts/decode-del-tep-capture.py b/scripts/decode-del-tep-capture.py new file mode 100644 index 0000000..ba61d64 --- /dev/null +++ b/scripts/decode-del-tep-capture.py @@ -0,0 +1,105 @@ +"""Decode the DeleteTagExtendedProperties (DelTep) WCF inBuff (HCAL R1.11 delete half). + +Reads the Run-B capture produced by scripts/Capture-DeleteTagExtendedProperties.ps1 and dumps the +DelTep WriteMessage body so the delete-request framing (tag name + property names + the +delete-from-server flag) can be read off and compared to the AddTEx serializer. + +Output is diagnostic. Sanitize before copying into docs/. +""" +import base64 +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CAPDIR = REPO_ROOT / "artifacts" / "reverse-engineering" / "instrumented-wcf-del-tep" +CAP = CAPDIR / "del-tep-capture-latest.ndjson" + +TAG = "RetestSdkWriteDelTepSdk" +PROP = "SdkDelProp" +OP_DEL = b"DelTep" +OP_GET = b"GetTepByNm" + + +def hexdump(label, buf, base=0): + print(f"=== {label}: {len(buf)} bytes ===") + for off in range(0, len(buf), 16): + c = buf[off:off + 16] + hp = " ".join(f"{x:02X}" for x in c) + ap = "".join(chr(x) if 32 <= x < 127 else "." for x in c) + print(f" {base + off:04X} {hp:<48} |{ap}|") + print() + + +def ascii_strings(buf, minlen=3): + out, cur, start = [], [], 0 + for i, x in enumerate(buf): + if 32 <= x < 127: + if not cur: + start = i + cur.append(chr(x)) + else: + if len(cur) >= minlen: + out.append((start, "".join(cur))) + cur = [] + if len(cur) >= minlen: + out.append((start, "".join(cur))) + return out + + +def u16_strings(buf, minlen=3): + out, i = [], 0 + while i < len(buf) - 1: + j, chars = i, [] + while j < len(buf) - 1 and 32 <= buf[j] < 127 and buf[j + 1] == 0: + chars.append(chr(buf[j])) + j += 2 + if len(chars) >= minlen: + out.append((i, "".join(chars))) + i = j + else: + i += 1 + return out + + +def main() -> int: + if not CAP.exists(): + print(f"Missing capture: {CAP}\nRun scripts/Capture-DeleteTagExtendedProperties.ps1 first.") + return 1 + + records = [] + for line in CAP.open(encoding="utf-8-sig"): + if line.strip(): + records.append(json.loads(line)) + + print(f"== {len(records)} MDAS bodies captured (Run B) ==") + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + flags = [] + if OP_DEL in body: + flags.append("DelTep") + if OP_GET in body: + flags.append("GetTepByNm") + if TAG.encode("ascii") in body or TAG.encode("utf-16-le") in body: + flags.append("TAG") + if PROP.encode("ascii") in body or PROP.encode("utf-16-le") in body: + flags.append("PROP") + print(f" [{idx:02d}] {rec.get('Phase'):26s} len={len(body):5d} {','.join(flags)}") + + print("\n== DelTep request(s) ==") + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + if rec.get("Phase") == "WCF.WriteMessage.Body" and OP_DEL in body: + hexdump(f"[{idx}] DelTep WriteMessage", body) + print(" UTF-16 strings:") + for off, s in u16_strings(body): + print(f" 0x{off:04X} {s!r}") + print(" ASCII strings:") + for off, s in ascii_strings(body): + print(f" 0x{off:04X} {s!r}") + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/AVEVA.Historian.Client/HistorianClient.cs b/src/AVEVA.Historian.Client/HistorianClient.cs index fa72656..7a07107 100644 --- a/src/AVEVA.Historian.Client/HistorianClient.cs +++ b/src/AVEVA.Historian.Client/HistorianClient.cs @@ -201,6 +201,15 @@ public sealed class HistorianClient : IAsyncDisposable return AddTagExtendedPropertiesAsync(tag, [new HistorianTagExtendedProperty(name, value ?? string.Empty)], cancellationToken); } + // Extended-property DELETE (DelTep) is intentionally NOT exposed publicly. Its wire format is + // captured and the serializer (HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest) is + // golden-verified against a server-accepted buffer, but the SDK cannot yet make the 2020 server + // accept the delete: the server's CHistStorage::DeleteTagExtendedProperties consults a + // per-connection working set that the native client populates by multiplexing GetTepByNm and + // DelTep over ONE connection, which the SDK's per-service WCF channels don't reproduce. See the + // documented-but-blocked path in HistorianWcfTagWriteOrchestrator and + // docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete. + /// /// Creates or updates the named tag in the Historian Runtime database via /// EnsureTags2. Currently only is diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs index dd5aa69..87296c8 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs @@ -108,6 +108,59 @@ internal static class HistorianTagExtendedPropertyProtocol return stream.ToArray(); } + /// + /// Serializes the DelTep (DeleteTagExtendedProperties) inBuff for a single tag's + /// properties identified by name. Same group framing as but + /// each property carries its name only (no 0x43 value variant) and the group trailer byte + /// is 0x00 (Add uses 0x01). Decoded from a native + /// DeleteTagExtendedPropertiesByName capture (see + /// docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete): + /// + /// uint32 groupCount (= 1) + /// byte 0x01 (group marker) + /// 0x09 + uint16 byteLen + ASCII tagName + /// uint32 propertyCount + /// repeated: byte 0x02 (property marker) + 0x09 + uint16 byteLen + ASCII propName + /// byte 0x00 (group trailer) + /// byte 0x00 (buffer terminator) + /// + /// The native deleteFromServer argument is not in the buffer — it is the client-side flag + /// that decides whether the wire op fires at all (the SDK always sends the server-delete form, + /// the only one that produces a DelTep call). + /// + public static byte[] SerializeDeleteRequest(string tagName, IReadOnlyList propertyNames) + { + ArgumentException.ThrowIfNullOrEmpty(tagName); + ArgumentNullException.ThrowIfNull(propertyNames); + if (propertyNames.Count == 0) + { + throw new ArgumentException("At least one property name is required.", nameof(propertyNames)); + } + + using MemoryStream stream = new(); + Span u32 = stackalloc byte[4]; + + BinaryPrimitives.WriteUInt32LittleEndian(u32, 1u); // group count + stream.Write(u32); + + stream.WriteByte(GroupMarker); + WriteCompactAscii(stream, tagName); + + BinaryPrimitives.WriteUInt32LittleEndian(u32, checked((uint)propertyNames.Count)); + stream.Write(u32); + + foreach (string name in propertyNames) + { + ArgumentException.ThrowIfNullOrEmpty(name, nameof(propertyNames)); + stream.WriteByte(PropertyMarker); + WriteCompactAscii(stream, name); // delete is by name only — no value variant + } + + stream.WriteByte(0x00); // group trailer (captured: 0x00 for delete, vs 0x01 for add) + stream.WriteByte(0x00); // buffer terminator + return stream.ToArray(); + } + private static void WriteCompactAscii(MemoryStream stream, string value) { byte[] ascii = Encoding.ASCII.GetBytes(value); diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs index bd47048..f239534 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs @@ -53,6 +53,29 @@ internal sealed class HistorianWcfTagWriteOrchestrator return Task.Run(() => AddTagExtendedProperties(tagName, properties), cancellationToken); } + /// + /// DelTep (DeleteTagExtendedProperties) — investigated but server-blocked, not wired to the + /// public API. The inBuff serializer is golden-verified against a server-accepted native + /// capture, and an SDK-added property is deletable (the native client read-syncs and deletes it). + /// But the SDK's own delete is rejected by CHistStorage::DeleteTagExtendedProperties + /// (SErrorException) even with matching Open2 mode/handle/inBuff and a GetTgByNm+GetTepByNm prime: + /// the server consults a per-connection working set the native client populates by multiplexing + /// GetTepByNm and DelTep over one connection, which the SDK's per-service WCF channels don't + /// reproduce. Kept for follow-up RE. See + /// docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete. + /// + internal Task DeleteTagExtendedPropertiesAsync( + string tagName, IReadOnlyList propertyNames, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tagName); + ArgumentNullException.ThrowIfNull(propertyNames); + if (propertyNames.Count == 0) + { + throw new ArgumentException("At least one property name is required.", nameof(propertyNames)); + } + return Task.Run(() => DeleteTagExtendedProperties(tagName, propertyNames), cancellationToken); + } + private bool EnsureTag(HistorianTagDefinition definition) { Guid contextKey = Guid.NewGuid(); @@ -126,8 +149,101 @@ internal sealed class HistorianWcfTagWriteOrchestrator return result; } - /// Env-gated dump of the clean AddTEx inBuff (base64) for golden-fixture capture, - /// mirroring the rename/SQL dump hooks — avoids hand-stitching MDAS chunk markers. + private bool DeleteTagExtendedProperties(string tagName, IReadOnlyList propertyNames) + { + Guid contextKey = Guid.NewGuid(); + var (histBinding, histEndpoint, _, _) = HistorianWcfBindingFactory.CreateBindingPair(_options); + Binding auxBinding = HistorianWcfBindingFactory.CreateAuxiliaryBinding(_options); + EndpointAddress statusEndpoint = HistorianWcfBindingFactory.CreateAuxiliaryEndpointAddress(_options, HistorianWcfServiceNames.Status); + EndpointAddress transactionEndpoint = HistorianWcfBindingFactory.CreateAuxiliaryEndpointAddress(_options, HistorianWcfServiceNames.Transaction); + EndpointAddress retrievalEndpoint = _options.Transport == HistorianTransport.LocalPipe + ? HistorianWcfBindingFactory.CreatePipeEndpointAddress(_options.Host, HistorianWcfServiceNames.Retrieval) + : HistorianWcfBindingFactory.CreateEndpointAddress(_options.Host, _options.Port, HistorianWcfServiceNames.Retrieval); + + bool result = false; + HistorianWcfAuthChainHelper.OpenAuthenticatedConnection( + _options, histBinding, histEndpoint, contextKey, CancellationToken.None, + connectionMode: HistorianWcfAuthChainHelper.NativeIntegratedWriteEnabledConnectionMode, + additionalSetup: (historyChannel, context) => + { + RunWritePriming(historyChannel, context, auxBinding, statusEndpoint, transactionEndpoint, retrievalEndpoint); + string handle = context.StorageSessionId.ToString("D").ToUpperInvariant(); + byte[] inBuff = HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest(tagName, propertyNames); + DumpTepIfRequested(inBuff); + // The server's CHistStorage::DeleteTagExtendedProperties resolves each property from + // the storage session's working set — so the tag identity (GetTgByNm) and its + // extended properties (GetTepByNm) must be fetched on THIS session first, exactly as + // the native client does (register → read → delete). The Retrieval prime channel is + // kept OPEN across the DelTep call (the registration is bound to the live channel + // session); without this prime the server throws SErrorException and DelTep returns + // false. See docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete. + result = PrimeThenDelete(auxBinding, retrievalEndpoint, handle, context.ClientHandle, tagName, () => + { + bool ok = historyChannel.DeleteTagExtendedProperties(handle, inBuff, out byte[] errorBuffer); + WriteDiag("DelTep", $"Returned={ok} Tag={tagName} PropCount={propertyNames.Count} InLen={inBuff.Length} ErrLen={errorBuffer?.Length ?? -1} ErrHex={(errorBuffer is null ? "" : Convert.ToHexString(errorBuffer))}"); + return ok; + }); + }); + return result; + } + + /// + /// Primes the Retrieval session before a DelTep delete exactly as the native client does: + /// register the tag identity (GetTgByNm) then fetch its extended properties + /// (GetTepByNm) so the server loads both into the storage session's working set, then runs + /// (the DelTep call) WHILE the Retrieval channel is still open. + /// Without this two-step prime — or if the channel is closed before DelTep — the server's + /// CHistStorage::DeleteTagExtendedProperties can't resolve the property and throws + /// SErrorException (DelTep returns false). The fetched rows are discarded — the calls are + /// for their server-side registration effect. See + /// docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete. + /// + private bool PrimeThenDelete( + Binding retrievalBinding, EndpointAddress retrievalEndpoint, string handle, uint clientHandle, + string tagName, Func deleteAction) + { + ChannelFactory factory = new(retrievalBinding, retrievalEndpoint); + HistorianWcfClientCredentialsHelper.Configure(factory, _options); + IRetrievalServiceContract4 channel = factory.CreateChannel(); + try + { + TryRun(() => channel.GetInterfaceVersion(out _)); + byte[] tagNames = HistorianTagExtendedPropertyProtocol.SerializeRequest(tagName); + + // Register the tag identity on this session (GetTgByNm, uint clientHandle). tagNames uses + // the same count+charCount+UTF-16 framing as GetTepByNm's request buffer. + try + { + uint tgSequence = 0; + uint tgRet = channel.GetTagInfosFromName(clientHandle, (uint)tagNames.Length, tagNames, ref tgSequence, out uint tgInfosSize, out byte[] tgInfos); + WriteDiag("DelTepPrime", $"GetTgByNm ret={tgRet} clientHandle={clientHandle} seq={tgSequence} infosSize={tgInfosSize} infosLen={tgInfos?.Length ?? -1}"); + } + catch (Exception ex) { WriteDiag("DelTepPrime", $"GetTgByNm threw: {ex.GetType().Name}: {ex.Message}"); } + + uint sequence = 0; + for (int page = 0; page < 64; page++) + { + bool ok; + byte[] responseBuffer; + try { ok = channel.GetTagExtendedPropertiesFromName(handle, tagNames, ref sequence, out responseBuffer, out byte[] errBuf); } + catch (Exception ex) { WriteDiag("DelTepPrime", $"GetTepByNm threw page={page}: {ex.GetType().Name}: {ex.Message}"); break; } + int rows = HistorianTagExtendedPropertyProtocol.ParseResponse(responseBuffer ?? []).Count; + WriteDiag("DelTepPrime", $"GetTepByNm page={page} ok={ok} respLen={responseBuffer?.Length ?? -1} rows={rows}"); + if (!ok) break; + if (rows == 0) break; + } + + // DelTep runs here, while the Retrieval prime channel is still open. + return deleteAction(); + } + finally + { + CloseSafely(channel, factory); + } + } + + /// Env-gated dump of the clean AddTEx / DelTep inBuff (base64) for golden-fixture + /// capture, mirroring the rename/SQL dump hooks — avoids hand-stitching MDAS chunk markers. private static void DumpTepIfRequested(byte[] inBuff) { string? path = Environment.GetEnvironmentVariable("AVEVA_HISTORIAN_TEP_DUMP"); diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs index 82ab6e2..1cd17ec 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs @@ -565,6 +565,12 @@ public sealed class HistorianClientIntegrationTests } } + // Extended-property DELETE (DelTep) has no live integration test: the wire format is captured and + // the serializer is golden-verified (WcfTagExtendedPropertyWriteProtocolTests), but the SDK + // cannot yet make the server accept the delete (the native single-connection working-set + // requirement isn't reproduced by per-service WCF channels). See + // docs/reverse-engineering/wcf-add-tag-extended-properties.md §Delete. + // Round-trip every live-verified analog data type + the non-default-range case. The // sandbox tag name is suffixed per case so the runs don't collide. Always cleans up. [Theory] diff --git a/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs b/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs index 65531a5..b7454b5 100644 --- a/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/StringHandleProbeDiagnosticTests.cs @@ -153,4 +153,43 @@ public sealed class StringHandleProbeDiagnosticTests } }); } + + /// + /// R1.3 / R1.4 reachability probe: ask GetSystemParameter (config) and GetRuntimeParameter (GETRP, + /// live runtime state) for timezone + event-storage-mode candidate keys. If the server timezone or + /// EventStorageMode surfaces through either named-parameter op, R1.3/R1.4 are deliverable on 2020; + /// if every candidate returns null, they are confirmed 2023R2/gRPC-only from the parameter angle. + /// Prints results — not an assertion test. + /// + [Fact] + public async Task TimezoneAndStorageMode_ParameterProbe_AgainstLocalHistorian() + { + if (!ShouldRun(out string host)) return; + + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe + }); + + // Timezone (R1.3) candidates + the one existing time-ish SystemParameter (TimeStampRule); + // EventStorageMode (R1.4) candidates + the existing EventStorage* params as controls. + string[] candidates = + [ + "TimeZone", "ServerTimeZone", "SystemTimeZone", "TimeZoneName", "SystemTimeZoneName", + "TimeStampRule", "ServerTime", + "EventStorageMode", "StorageMode", "EventStorage", "EventStorageDuration", + "HistorianVersion", // known-good control + ]; + + foreach (string key in candidates) + { + string? sys = null, run = null; + string sysErr = "", runErr = ""; + try { sys = await client.GetSystemParameterAsync(key); } catch (Exception ex) { sysErr = ex.GetType().Name; } + try { run = await client.GetRuntimeParameterAsync(key); } catch (Exception ex) { runErr = ex.GetType().Name; } + _output.WriteLine($"{key,-22} SystemParameter={(sys is null ? "" : $"\"{sys}\"")}{(sysErr.Length > 0 ? $" [{sysErr}]" : "")} RuntimeParameter={(run is null ? "" : $"\"{run}\"")}{(runErr.Length > 0 ? $" [{runErr}]" : "")}"); + } + } } diff --git a/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs index 0597eb0..097a752 100644 --- a/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs @@ -69,4 +69,58 @@ public sealed class WcfTagExtendedPropertyWriteProtocolTests Assert.Throws(() => HistorianTagExtendedPropertyProtocol.SerializeAddRequest("T", Array.Empty())); } + + // Server-accepted DelTep inBuff for tag "RetestSdkWriteDelTepSdk", property "SdkDelProp", + // extracted byte-for-byte from the native DeleteTagExtendedPropertiesByName WriteMessage in the + // cross-session capture (scripts/Capture-DeleteTagExtendedProperties.ps1) — DelTep returned + // success, so it is server-validated. + private const string ServerAcceptedDeleteInBuffBase64 = + "AQAAAAEJFwBSZXRlc3RTZGtXcml0ZURlbFRlcFNkawEAAAACCQoAU2RrRGVsUHJvcAAA"; + + [Fact] + public void SerializeDeleteRequest_MatchesServerAcceptedBuffer() + { + byte[] expected = Convert.FromBase64String(ServerAcceptedDeleteInBuffBase64); + byte[] actual = HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest( + "RetestSdkWriteDelTepSdk", ["SdkDelProp"]); + Assert.Equal(expected, actual); + } + + [Fact] + public void SerializeDeleteRequest_SingleProperty_HasExpectedLayout() + { + byte[] buf = HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest( + "ReactorTemp", ["Location"]); + + int c = 0; + Assert.Equal(1u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(c, 4))); c += 4; // group count + Assert.Equal(0x01, buf[c++]); // group marker + Assert.Equal(0x09, buf[c++]); // compact string marker + Assert.Equal(11, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; // tag byte len + Assert.Equal("ReactorTemp", Encoding.ASCII.GetString(buf.AsSpan(c, 11))); c += 11; + Assert.Equal(1u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(c, 4))); c += 4; // property count + Assert.Equal(0x02, buf[c++]); // property marker + Assert.Equal(0x09, buf[c++]); + Assert.Equal(8, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; + Assert.Equal("Location", Encoding.ASCII.GetString(buf.AsSpan(c, 8))); c += 8; + // Delete carries NO value variant (Add would have 0x43 ... here). + Assert.Equal(0x00, buf[c++]); // group trailer (0x00 for delete, vs 0x01 for add) + Assert.Equal(0x00, buf[c++]); // buffer terminator + Assert.Equal(buf.Length, c); + } + + [Fact] + public void SerializeDeleteRequest_MultipleNames_EncodesCount() + { + byte[] buf = HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest("T", ["A", "B"]); + // group count @0, then 0x01 marker, 0x09 + u16(1) + "T" => property count at offset 4+1+1+2+1 = 9 + Assert.Equal(2u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(9, 4))); + } + + [Fact] + public void SerializeDeleteRequest_NoProperties_Throws() + { + Assert.Throws(() => + HistorianTagExtendedPropertyProtocol.SerializeDeleteRequest("T", Array.Empty())); + } } diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index c135f06..1273734 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -390,27 +390,67 @@ internal static class Program groupType.GetMethod("Add", new[] { propType })!.Invoke(group, [prop]); listType.GetMethod("Add", new[] { groupType })!.Invoke(list, [group]); - MethodInfo addTepMethod = accessType.GetMethods() - .First(m => m.Name == "AddTagExtendedProperties" && m.GetParameters().Length == 2); - object addTepError = Activator.CreateInstance(errorType)!; - object?[] addTepArgs = [list, addTepError]; - WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tep"); - bool addTepOk = false; - string? addTepEx = null; - try { addTepOk = (bool)addTepMethod.Invoke(access, addTepArgs)!; } - catch (TargetInvocationException ex) { addTepEx = FormatException(ex.InnerException ?? ex); } - tepRows.Add(new + // 2b) Add the property (AddTEx) unless --tep-skip-add. Skipped in the second + // (delete) session so the local cache isn't re-seeded with an UNSYNCED entry — + // DeleteTagExtendedPropertiesByName returns err 229 against unsynced properties. + if (!HasFlag(args, "--tep-skip-add")) { - Kind = "AddTagExtendedProperties", - Success = addTepOk, - Exception = addTepEx, - ErrorDescription = GetPropertyText(addTepArgs[1]!, "ErrorDescription"), - ErrorCode = GetPropertyText(addTepArgs[1]!, "ErrorCode"), - }); + MethodInfo addTepMethod = accessType.GetMethods() + .First(m => m.Name == "AddTagExtendedProperties" && m.GetParameters().Length == 2); + object addTepError = Activator.CreateInstance(errorType)!; + object?[] addTepArgs = [list, addTepError]; + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tep"); + bool addTepOk = false; + string? addTepEx = null; + try { addTepOk = (bool)addTepMethod.Invoke(access, addTepArgs)!; } + catch (TargetInvocationException ex) { addTepEx = FormatException(ex.InnerException ?? ex); } + tepRows.Add(new + { + Kind = "AddTagExtendedProperties", + Success = addTepOk, + Exception = addTepEx, + ErrorDescription = GetPropertyText(addTepArgs[1]!, "ErrorDescription"), + ErrorCode = GetPropertyText(addTepArgs[1]!, "ErrorCode"), + }); + } - // 3) Optional delete (DelTep) to capture its inBuff too. + // 3) Optional delete (DelTep) to capture its inBuff too. The native + // DeleteTagExtendedPropertiesByName performs a CLIENT-SIDE sync check and rejects + // (err 229) any property the local cache doesn't see as server-synchronized — so a + // just-added property can't be deleted in the same session. To capture DelTep: + // Run A: add-tep --tep-tag RetestSdkWriteDelTepSdk --tep-name P --tep-value V + // Run B: add-tep --tep-tag RetestSdkWriteDelTepSdk --tep-name P \ + // --tep-skip-create --tep-skip-add --tep-delete + // Run B opens a FRESH connection, then we fetch the property from the server + // (GetTagExtendedPropertiesByName, fetchFromServer=true) to seed the local cache as + // SYNCED before deleting — which lets DeleteTagExtendedPropertiesByName reach the wire. if (HasFlag(args, "--tep-delete")) { + // Seed the synced local cache: prime tag identity + force a server fetch. + MethodInfo? primeForDelete = accessType.GetMethods() + .FirstOrDefault(m => m.Name == "GetTagInfoByName" && m.GetParameters().Length == 4); + if (primeForDelete is not null) + { + object tibErr = Activator.CreateInstance(errorType)!; + object?[] tibArgs = [tepTag, true, null, tibErr]; + try { primeForDelete.Invoke(access, tibArgs); } catch { } + } + MethodInfo? readForSync = accessType.GetMethods() + .FirstOrDefault(m => m.Name == "GetTagExtendedPropertiesByName" && m.GetParameters().Length == 4); + string? syncRead = null; + if (readForSync is not null) + { + object syncErr = Activator.CreateInstance(errorType)!; + object?[] syncArgs = [tepTag, true, null, syncErr]; + try + { + bool syncOk = (bool)readForSync.Invoke(access, syncArgs)!; + syncRead = $"GetTagExtendedPropertiesByName(fetch)={syncOk} err={GetPropertyText(syncArgs[3]!, "ErrorDescription")}"; + } + catch (TargetInvocationException ex) { syncRead = "read-for-sync threw: " + FormatException(ex.InnerException ?? ex); } + } + tepRows.Add(new { Kind = "ReadForSync", Detail = syncRead }); + MethodInfo? delTepMethod = accessType.GetMethods() .FirstOrDefault(m => m.Name == "DeleteTagExtendedPropertiesByName" && m.GetParameters().Length == 4); if (delTepMethod is not null) @@ -420,10 +460,11 @@ internal static class Program namesColType.GetMethod("Add", new[] { typeof(string) })!.Invoke(names, [propName]); object delErr = Activator.CreateInstance(errorType)!; object?[] delArgs = [tepTag, names, true, delErr]; + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-del-tep"); bool delOk = false; string? delEx = null; try { delOk = (bool)delTepMethod.Invoke(access, delArgs)!; } catch (TargetInvocationException ex) { delEx = FormatException(ex.InnerException ?? ex); } - tepRows.Add(new { Kind = "DeleteTagExtendedPropertiesByName", Success = delOk, Exception = delEx, ErrorDescription = GetPropertyText(delArgs[3]!, "ErrorDescription") }); + tepRows.Add(new { Kind = "DeleteTagExtendedPropertiesByName", Success = delOk, Exception = delEx, ErrorDescription = GetPropertyText(delArgs[3]!, "ErrorDescription"), ErrorCode = GetPropertyText(delArgs[3]!, "ErrorCode") }); } }