From 5ce62a5900146f84c07670562ba6eed0d37e6b43 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 4 May 2026 22:04:27 -0400 Subject: [PATCH] Wire ApplyScaling, StorageRate; close out write-commands plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ApplyScaling (HistorianTagDefinition.ApplyScaling): The EnsT2 trailer's second byte controls server-side scaling — `FE 00` mirrors MinRaw to MinEU and sets AnalogTag.Scaling=0; `FE 01` persists distinct MinRaw/MaxRaw and sets Scaling=1. Decoded by toggling set_ApplyScaling on the native harness and capturing the wire bytes for both values with identical inputs. The earlier docs claimed EnsureTagAsync needed a follow-up "UpdateTags" call; the WCF surface has no such operation — toggling that one byte is the whole fix. StorageRate (HistorianTagDefinition.StorageRateMs): Serializer accepts a non-default rate, validated empirically against the live server which only accepts quantized values (1000/5000/10000/60000/300000 ms). EnsureTagAsync upsert semantics: Second call on the same tag name with different fields succeeds and updates Description, MinEU, MaxEU, MinRaw, MaxRaw, Scaling in place (verified by direct SQL inspection in a live test). Plan + doc closeout: write-commands-reverse-engineering.md rewritten as a current-state plan with three workstreams (A doc closeout / B idempotency / C1 StorageRate) and a parallelism table; prior phase notes preserved as appendix. handoff.md, implementation-status.md, wcf-contract-evidence.md, README.md updated to remove "writes are out of scope" / non-existent UpdateTags references and document the actual EnsT2 wire format including the `FE xx` trailer. Reverse-engineering harness gains --write-apply-scaling and a SQL post-check that prints the persisted AnalogTag bounds so future RE sessions can verify wire→DB causality without leaving the harness. 169/169 tests pass (was 165; +4 new tests covering ApplyScaling, StorageRate golden bytes, StorageRate live persistence, and EnsureTagAsync upsert semantics). Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 6 +- README.md | 24 +- .../write-commands-reverse-engineering.md | 831 ++++-------------- docs/reverse-engineering/handoff.md | 21 +- .../implementation-status.md | 14 +- .../wcf-contract-evidence.md | 32 +- .../Models/HistorianTagDefinition.cs | 33 +- .../Wcf/HistorianTagWriteProtocol.cs | 32 +- .../Wcf/HistorianWcfTagWriteOrchestrator.cs | 4 +- .../AVEVA.Historian.Client.Tests.csproj | 1 + .../HistorianClientIntegrationTests.cs | 173 ++++ .../HistorianTagWriteProtocolTests.cs | 89 +- .../Program.cs | 22 +- 13 files changed, 561 insertions(+), 721 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index edc39e1..713fee7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ Reads (the original required surface, all working live as of 2026-05-04): Writes (added 2026-05-04 by explicit user request — do not extend further without one): -- `EnsureTagAsync` for analog types: Float, Double, Int2, Int4, UInt4 (live-verified end-to-end). Other types (SingleByteString/DoubleByteString/Int1/Int8/UInt8) fail at native AddTag — likely require a different path and are intentionally not supported. `MinEU`/`MaxEU` round-trip correctly into the DB; `MinRaw`/`MaxRaw` are sent on the wire but the server mirrors them to MinEU/MaxEU when ApplyScaling=false (verified against native — server quirk, not SDK bug). +- `EnsureTagAsync` for analog types: Float, Double, Int2, Int4, UInt4 (live-verified end-to-end). Other types (SingleByteString/DoubleByteString/Int1/Int8/UInt8) fail at native AddTag — likely require a different path and are intentionally not supported. `MinEU`/`MaxEU`/`MinRaw`/`MaxRaw` all round-trip into the DB. By default `ApplyScaling=false` and the server mirrors MinRaw→MinEU and sets `AnalogTag.Scaling=0`; set `ApplyScaling=true` on the definition to persist distinct raw bounds with `AnalogTag.Scaling=1`. The wire encoding is the trailer's second byte (`FE 00` vs `FE 01`). - `DeleteTagAsync` `AddS2` (write samples) is architecturally blocked — server cache only ingests from configured IOServers/ApplicationServer pipelines. Do not add write-samples support. @@ -87,7 +87,7 @@ End-to-end chain working from a pure managed .NET 10 client: `Hist.GetV → Hist ### Write-path notes (added 2026-05-04) -`EnsureTagAsync` and `DeleteTagAsync` chain follow the same pattern as reads but require Open2 with `NativeIntegratedWriteEnabledConnectionMode = 0x401` (Process | Write | IntegratedSecurity) — the read-path's `0x402` (read-only) makes the server return err 132 `OperationNotEnabled` silently. The analog Float `CTagMetadata` payload is 144 bytes with a leading `0x4E` marker byte and a 2-byte trailer `FE 00`. See `docs/reverse-engineering/handoff.md` and the `WriteDiag` env-gated diagnostic helper in `HistorianWcfTagWriteOrchestrator` for capture details. +`EnsureTagAsync` and `DeleteTagAsync` chain follow the same pattern as reads but require Open2 with `NativeIntegratedWriteEnabledConnectionMode = 0x401` (Process | Write | IntegratedSecurity) — the read-path's `0x402` (read-only) makes the server return err 132 `OperationNotEnabled` silently. The analog Float `CTagMetadata` payload is 144 bytes with a leading `0x4E` marker byte and a 2-byte trailer `FE xx` where the second byte is the ApplyScaling flag (`00` for false / `01` for true). The `IHistoryServiceContract2` surface has no `UpdateTags` operation — distinct MinRaw/MaxRaw persistence is achieved entirely by toggling that one byte in the EnsT2 payload, not via a follow-up call. See `docs/reverse-engineering/handoff.md` and the `WriteDiag` env-gated diagnostic helper in `HistorianWcfTagWriteOrchestrator` for capture details. ### Remaining gaps @@ -96,7 +96,7 @@ Smaller, isolated items — none block the production read surface: - Remote TCP transports verified by pointing `HISTORIAN_REMOTE_TCP_HOST` (and `HISTORIAN_REMOTE_TCPCERT_HOST` for the cert variant) at the host's own LAN IP — exercises the `MdasNetTcpWindows` / `MdasNetTcpCertificate` binding branches and SSPI/TLS handshake against a hostname rather than the loopback fast path. `RemoteTcpIntegrated`: 9 tests (Probe + full read surface + status helpers). `RemoteTcpCertificate`: Probe only; deeper coverage awaits an explicit-creds setup. True off-box verification (e.g. Linux client) would require porting `HistorianSspiClient` off `InitializeSecurityContextW` to managed `NegotiateAuthentication` + GSSAPI. - Explicit username/password tag-metadata path is wired (validator only blocks no-auth-at-all), but live-verification requires `HISTORIAN_USER`+`HISTORIAN_PASSWORD` set; gated test `GetTagMetadataAsync_ExplicitCredentials_AgainstLocalHistorian` skips otherwise. - Per-row trailing 35 bytes of `GetNextQueryResultBuffer` are now mapped (see `HistorianDataQueryProtocol.TryParseGetNextQueryResultBufferRows` doc comment) — bytes 3-10 = duplicate FILETIME (already used by aggregate parser), bytes 0-2 + 19-34 = server-internal sample/storage metadata with no clear user-facing meaning. No new public fields added; revisit if a customer asks for storage metadata exposure. -- `EnsureTagAsync` distinct `MinRaw`/`MaxRaw` persistence requires `ApplyScaling=true` + a follow-up `UpdateTags` call — not yet wired (no API user has asked). +- (No remaining gaps in the write surface — `ApplyScaling` is now wired, see Required SDK Surface above.) ### Tools Layer diff --git a/README.md b/README.md index 8766e6d..84b3eea 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@ production SDK has no dependency on `aahClientManaged.dll`, `aahClient.dll`, or any other AVEVA native runtime — the wire protocol is reverse-engineered and re-implemented in C#. -Read-only by design. The required surface (per [`CLAUDE.md`](CLAUDE.md)): +The supported surface (per [`CLAUDE.md`](CLAUDE.md)): | Operation | Status | |---|---| | `ProbeAsync` | live-verified | | `ReadRawAsync` | live-verified | -| `ReadAggregateAsync` | live-verified (TimeWeightedAverage; other modes need fixtures) | +| `ReadAggregateAsync` | live-verified across all 16 retrieval modes | | `ReadAtTimeAsync` | live-verified | | `ReadEventsAsync` | live-verified (typed event + 31-property property bag) | | `BrowseTagNamesAsync` | live-verified | @@ -19,8 +19,13 @@ Read-only by design. The required surface (per [`CLAUDE.md`](CLAUDE.md)): | `GetConnectionStatusAsync` | synthesized from authenticated probe (matches native semantic) | | `GetStoreForwardStatusAsync` | synthesized defaults (no SF sidecar to probe) | | `GetSystemParameterAsync` | live-verified via `Stat/GetSystemParameter` | +| `EnsureTagAsync` | live-verified for analog Float/Double/Int2/Int4/UInt4; `ApplyScaling=true` persists distinct MinRaw/MaxRaw | +| `DeleteTagAsync` | live-verified | -Out of scope: write-back, store-forward write, configuration changes. +Out of scope: writing samples (`AddS2` is architecturally blocked — the server's +runtime cache only ingests from configured IOServer / Application Server +pipelines), store-forward write, configuration changes, discrete/string tag +creation (native `AddTag` rejects them). ## Quick start @@ -160,9 +165,10 @@ property dictionary → Retr.EndEventQuery → Hist.Close2 ## Status -124 unit + live integration tests pass (`dotnet test --logger "console;verbosity=minimal"`). -Full read-only SDK surface verified end-to-end against both a local Historian -(`LocalPipe`) and a remote Historian (`RemoteTcpIntegrated` over Net.TCP with -Windows transport auth). `RemoteTcpCertificate` ProbeAsync is live-verified; -the other ops over the certificate transport plus the explicit-credentials -path await live verification. +165 unit + live integration tests pass (`dotnet test --logger "console;verbosity=minimal"`). +Full SDK surface — reads, browse, metadata, status, plus the two write ops +(`EnsureTagAsync` / `DeleteTagAsync`) — verified end-to-end against both a +local Historian (`LocalPipe`) and a remote Historian (`RemoteTcpIntegrated` +over Net.TCP with Windows transport auth). `RemoteTcpCertificate` ProbeAsync +is live-verified; deeper coverage over the cert transport plus the +explicit-credentials path await additional verification. diff --git a/docs/plans/write-commands-reverse-engineering.md b/docs/plans/write-commands-reverse-engineering.md index 0ad9d1d..7b568af 100644 --- a/docs/plans/write-commands-reverse-engineering.md +++ b/docs/plans/write-commands-reverse-engineering.md @@ -1,241 +1,133 @@ # Plan: Reverse-Engineering Write Commands -Status: **PHASE 2 PARTIALLY EXECUTED on 2026-05-04** — write-scenario -harness extension built and captured the full EnsT2(Float) wire byte -sequence against a real sandbox tag. AddS2 is blocked client-side by -"Tag not added to server" (error 168) — the native `AddStreamedValue` -refuses to send because the tag isn't in the server's session cache, -even though `EnsT2` created it in the Runtime DB. AddS2 wire bytes -**not yet captured**; needs a separate session to resolve the -post-EnsT2 registration prereq (likely RTag2 with the analog tag GUID, -mirroring the event flow's RTag2(CmEventTagId)). +## Status (2026-05-04, post-ApplyScaling landing) -## Phase 2 results +Phase 2 is **complete**. The write surface that the server actually +supports for managed clients is implemented and live-verified: -**Sandbox tag created** in Runtime DB: `RetestSdkWriteSandbox`, -wwTagKey=240, DateCreated=2026-05-04 07:49:50. Single dedicated tag -per safety §1; no other tags touched. +- `EnsureTagAsync` for analog tags (Float, Double, Int2, Int4, UInt4), + with optional `ApplyScaling=true` for distinct MinRaw/MaxRaw + persistence (`AnalogTag.Scaling=1`). +- `DeleteTagAsync` for any tag created via the SDK. -**`tools/AVEVA.Historian.NativeTraceHarness/Program.cs` extended** with -`--scenario write`: +Architecturally blocked / out of scope: -- New args: `--write-sandbox-tag ` (default - `RetestSdkWriteSandbox`; refuses any name that doesn't start with - `RetestSdkWrite`), `--write-value ` (default 42.5), - `--write-data-type ` (default Float), `--write-delete-after` - (best-effort cleanup). -- Toggles `ConnectionArgs.ReadOnly` to false when scenario is `write` - (otherwise the connection rejects writes with error 132 "Operation - is not enabled"). -- Calls `ArchestrA.HistorianAccess.AddTag` (drives `EnsT2` on the wire), - then `ArchestrA.HistorianAccess.AddStreamedValue` (would drive - `AddS2` but currently aborts client-side at error 168). -- Resolves the actual `wwTagKey` via SQL when `AddTag` returns 0 - because the tag already exists from a prior session. -- Public `AddStreamedValue` overload selector: instance method whose - signature is `(HistorianDataValue, …, HistorianAccessError&)` — - picks the simplest dispatcher that's actually reflectable (the - 4-param impl is private and not visible to reflection). +- **`AddS2` (write samples)** — server's runtime cache only ingests + from configured IOServers / Application Server pipelines, not from + client-only `AddTag` flows. The native wrapper hits the same wall; + this is a server architecture decision, not a protocol gap. +- **Discrete / String / Int1 / Int8 / UInt8 tag types** — fail at + native `HistorianAccess.AddTag` before any wire bytes leave the + client. Likely require a different code path (`AddTagExtendedProperties` + or pre-population via SMC); not investigated. -**Captures landed** at -`artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/bothmessage-write-capture-latest.ndjson` -(46 records: 23 outgoing + 23 incoming). Same priming chain as the -event flow: +This plan covers the residual workstreams. -``` -Hist.GetV → Hist.GetI ×2 → Hist.ValCl ×2 → Hist.Open2 → -Stat.GetV ×2 → Stat.GETHI ×2 → Hist.UpdC3 → -Stat.GetSystemParameter ×7 → Trx.GetV → Stat.GetV → Retr.GetV → -Hist.EnsT2(Float) → Hist.Close2 -``` +## Workstreams -No `RTag2`. The chain identical to the event flow except the EnsT2 -payload is the analog CTagMetadata instead of the event one, and there -is NO RTag2 between Open2 and EnsT2 (events used RTag2 to register -`CmEventTagId`). +### A. Documentation closeout -**Native EnsT2(Float) request body** (record 42, 322 bytes total; the -146-byte CTagMetadata `InBuff` payload is the new evidence target): +Status docs across the repo still describe write commands as +"in progress" or speculate about a non-existent `UpdateTags` +operation. Closing those out so future agents don't re-walk the +same dead ends. -```text -67 03 00 01 00 00 00 04 C6 02 01 00 00 00 00 00 -00 00 00 00 00 00 00 00 00 00 00 09 15 00 52 65 -74 65 73 74 53 64 6B 57 72 69 74 65 53 61 6E 64 -62 6F 78 FF FF FF FF FF FF FF FF FF FF FF FF FF -FF FF FF 09 18 00 53 44 4B 20 77 72 69 74 65 2D -52 45 20 73 61 6E 64 62 6F 78 20 74 61 67 09 04 -00 4D 44 41 53 02 01 01 00 00 00 01 E8 03 00 00 -D6 00 0E 4F BC DB DC 01 1A 03 09 04 00 74 65 73 -74 10 27 00 00 00 00 00 00 00 00 F0 3F FE 00 01 -01 01 -``` +| Step | File | Action | +|---|---|---| +| A1 | `docs/reverse-engineering/handoff.md` | Add a "Write-flow status" note marking EnsT2/DelT live; remove stale write-blocker callouts | +| A2 | `docs/reverse-engineering/implementation-status.md` | Flip EnsT2 / DelT rows from "out of scope" to "implemented"; add ApplyScaling row | +| A3 | `docs/reverse-engineering/wcf-contract-evidence.md` | Add evidence rows for `EnsT2(analog)` and `DelT` pointing at the captured fixtures | +| A4 | `README.md` | Operation-status table reflects the two write ops | -Visible fields (still being decoded against the -`CTagUtil.ConvertTagMetadataToHistorianTag` IL at token `0x060055CE`): +### B. EnsT2 idempotency / update behavior -- `09 15 00 RetestSdkWriteSandbox` (compact ASCII tag name, len 21) -- 16 bytes of `FF` — possibly a placeholder/sentinel for `CommonArchestraEventTypeId`-equivalent that's not used for analog -- `09 18 00 SDK write-RE sandbox tag` (compact ASCII description, len 24) -- `09 04 00 MDAS` (compact ASCII metadata provider) -- `09 04 00 test` (compact ASCII engineering unit) -- `0E 4F BC DB DC 01 1A 03` byte-pattern looks like an Int64 FILETIME (date-created ~2026) -- `10 27 00 00` = uint32 0x2710 = 10000 (storage-related) -- `00 00 00 00 00 00 F0 3F` = double 1.0 (likely IntegralDivisor or similar scaling) -- `FE 00 01 01 01` = trailer (matches event tag's `2F 27 01 01 01` shape) +We don't currently know what happens when `EnsureTagAsync` is called +against a tag name that already exists with different fields. Three +plausible outcomes: server errors, server silently updates, server +no-ops. This affects how callers should think about the API +(create-only vs upsert). -**Decoder script** at `scripts/decode-write-capture.py` for the next -session. +| Step | Action | +|---|---| +| B1 | Add a live integration test that calls `EnsureTagAsync` twice on the same tag name with different `MinEU/MaxEU/Description`; query SQL after each call to capture observed behavior | +| B2 | Document the observed contract in `HistorianTagDefinition` doc-comment and (if surprising) in CLAUDE.md | -## Phase 2 follow-on findings (2026-05-04, second pass) +### C. Expose currently-hardcoded CTagMetadata fields -**The AddS2 prereq is architectural, not protocol-level.** Three follow-up -attempts to trigger AddS2 from the sandbox harness all hit a client-side -gate before any AddS2 byte reaches the wire: +The serializer hardcodes `StorageRate=1000ms`, `StorageType=Cyclic`, +`IntegralDivisor=1.0`, and a few flag-block bytes. The server accepts +those defaults so existing tests pass, but customers building tags +with non-default rates can't currently express that. -1. **TagKey synthetic→real override.** First attempt used the placeholder - `TagKey=10000000` returned by `HistorianAccess.AddTag`. Native - `AddStreamedValue` refused with error 168 "Tag not added to server". - The harness now ALWAYS resolves the real `wwTagKey` from - `Runtime.dbo.Tag` after AddTag (logged as `TagKeyOverride: Synthetic→RealFromSql`). - Result: error code shifts to **129 "Tag not found in cache"**. +| Step | Field | Effort | Notes | +|---|---|---|---| +| C1 | `StorageRate` (uint32 ms) | small — wire field is already at a known offset, just plumb a parameter through | Default stays 1000ms | +| C2 | `StorageType` (Cyclic / Delta) | medium — need a comparison capture to find which byte in the flag block encodes it | Deferred unless customer asks | +| C3 | `IntegralDivisor` (double) | small — wire field already known | Deferred unless customer asks | -2. **Server-cache settle wait.** Inserted up to 8s sleep between AddTag and - AddStreamedValue (configurable via `--write-resync-wait-seconds`). The - wait period contains 2× UpdC3 + 2× Trx/GetV keep-alives but no - server-side cache update — error 129 persists. +C1 is the only one I'm executing in this round. C2/C3 are listed +for completeness; pick them up when there's a concrete request. -3. **Fresh process / fresh connection.** Skipped AddTag entirely - (`--write-skip-add-tag`) and ran AddStreamedValue alone against the - already-existing sandbox tag. New native client instance, new - client-side cache, new server session. **Same error 129 — no AddS2 - bytes sent on wire.** Capture confirms 44 records ending in Close2. +### D. Deferred — no current evidence or customer ask -**Interpretation.** The Historian engine's runtime tag cache only -ingests tags from configured IOServers / Application Server data pipelines, -not from `HistorianAccess.AddTag`-only client flows. `HistorianAccess.AddTag` -populates `Runtime.dbo.Tag` (we confirmed wwTagKey=240 was created) but -does not register the tag with the live cache that `AddStreamedValue` -checks. That registration happens server-side when an upstream data -producer (an OPC driver, the AnE event subsystem, the Application Server -attribute store, etc.) claims the tag. +| ID | Item | Why deferred | +|---|---|---| +| D1 | `AddTagExtendedProperties` / `DeleteTagExtendedProperties` | No wire captures yet; no customer ask | +| D2 | `AddRevisionValuesBegin/Value/End` (revision-write path) | Multi-step capture needed against an existing historized tag; complex; no customer ask | +| D3 | Discrete/String/Int1/Int8/UInt8 EnsT2 root cause | Native `AddTag` fails for these — likely requires an entirely different code path; would need a fresh capture and IL walkthrough | -For SDK purposes this means **`WriteValueAsync` cannot be implemented as -a generic client API against this server architecture.** The SDK's writeable -surface is realistically: +## Parallelism -- ✅ `EnsureTagAsync` (drives EnsT2 — 146-byte payload captured) -- ✅ `DeleteTagAsync` (drives DelT — not yet captured but should be straightforward) -- ❌ `WriteValueAsync` — won't work as designed; the server gates the - data path on tags being live in its in-memory cache -- ❓ `WriteRevisionAsync` — `HistorianAccess.AddRevisionValuesBegin/Value/End` - may use a different code path (intended for editing existing historized - data); needs a separate capture against an existing tag with stored history +| Track | Files touched | Conflicts with | +|---|---|---| +| A1 | `docs/reverse-engineering/handoff.md` | none | +| A2 | `docs/reverse-engineering/implementation-status.md` | none | +| A3 | `docs/reverse-engineering/wcf-contract-evidence.md` | none | +| A4 | `README.md` | none | +| B | `tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs`, `src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs` | C1 (same `HistorianTagDefinition` file) | +| C1 | `src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs`, `src/AVEVA.Historian.Client/Wcf/HistorianTagWriteProtocol.cs`, `src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs`, `tests/AVEVA.Historian.Client.Tests/HistorianTagWriteProtocolTests.cs`, `tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs` | B (same `HistorianTagDefinition` and integration-test file) | -Phase 2 effective deliverables: +**Concurrency-safe groupings:** -- ✅ NativeTraceHarness `--scenario write` extension -- ✅ EnsT2(Float) 146-byte CTagMetadata wire bytes -- ✅ Sandbox tag `RetestSdkWriteSandbox` in Runtime DB (wwTagKey=240) -- ⏸ AddS2 — blocked architecturally; **not just a protocol gap** -- ⏸ DelT — not yet captured (need `--write-delete-after` run) -- ⏸ Revision write path — separate capture needed against a historized - tag +- A1–A4 are pairwise independent and pairwise independent of B and C1 → can be assigned to four agents in parallel. +- B and C1 both touch `HistorianTagDefinition.cs`. Sequence them: B first (it just adds a doc-comment) then C1 (which adds a field). -## Phase 3 partial (2026-05-04) — EnsureTagAsync live, DeleteTagAsync partial +In a single-agent execution (this session), the order is: A* batched +edits → B → C1 → build/test → commit. -`HistorianTagWriteProtocol` + `HistorianWcfTagWriteOrchestrator` + -`HistorianClient.EnsureTagAsync`/`DeleteTagAsync` landed: +## Success Criteria -- `HistorianTagDefinition` public model (TagName/Description/EngineeringUnit/ - DataType/MinEU/MaxEU; only `Float` data type currently supported live). -- `HistorianTagWriteProtocol.SerializeAnalogCTagMetadata` — produces 146-byte - payload byte-for-byte identical to the captured native EnsT2(Float) request. -- `HistorianTagWriteProtocol.SerializeDeleteTagNames` — `[ushort 0x6751, - ushort 1, uint count, per-tag (uint charCount + UTF-16 chars)]`. -- `HistorianWcfTagWriteOrchestrator` — both EnsT2 and DelT run the full - Stat-priming chain captured for the analog flow (UpdC3 + Stat.GetV ×3 + - Stat.GETHI ×2 + 7× GetSystemParameter + Trx.GetV + Retr.GetV). -- New tag-origin marker `0xC7` added to `MapDataType` (SDK-created tags have - byte 1 = 0xC7, distinct from 0xCF system / 0xC3 MDAS-routed). +- A1–A4: documentation reflects the actual current write surface and + removes references to non-existent operations (`UpdateTags`). +- B: a passing live test that asserts the observed double-EnsT2 + behavior, plus a doc-comment update on `HistorianTagDefinition`. +- C1: `HistorianTagDefinition.StorageRateMs` field exposed, default + preserves existing wire output, golden test for a non-default rate, + live test that creates a tag with a non-default rate and asserts + via SQL that `Tag.CurrentEditorUserKey` (or the storage-rate column, + TBD) reflects the value. +- All 165+ tests still pass; no regression in existing live tests. -Golden-byte tests (5): EnsT2(Float) byte-for-byte match against the captured -146-byte fixture; DelT(single tag) byte-for-byte; DelT(multi-tag); empty list -throws; different-inputs-produce-different-bytes. +--- -Live integration test -(`EnsureTagAsync_AndDeleteTagAsync_RoundTrip_AgainstLocalHistorian`, -gated by `HISTORIAN_WRITE_SANDBOX_TAG=RetestSdkWriteSandbox`): EnsureTagAsync -followed by GetTagMetadataAsync confirms the sandbox tag is created in -the Runtime DB. Test passes 130/130 in the full suite. +## Appendix: Prior Phase Notes -**Known DelT gap.** SDK's DeleteTagAsync currently returns true but the -server-side cascading deletion does not always complete — the row remains -in `Runtime.dbo.Tag` even after the call returns. The captured native flow's -DelT removes the tag cleanly (verified via the harness with -`--write-delete-after`), so something the native code does between or -around the WCF DelT call is missing from our orchestrator. The harness -cleanup path remains the documented workaround for sandbox housekeeping. +The historical phase logs that drove the implementation are preserved +below for context. They describe the path from "write surface +unknown" to "write surface implemented and live-verified" as it +unfolded through 2026-05-04. Anything in this appendix that contradicts +the current Status section above is superseded. -## DelT investigation findings (2026-05-04) +### Phase 1 findings (recorded, not implementing) -Investigation step 1 — wire-byte parity check: the captured native DelT -request sends `ref` input values `statusSize=1` + `status=null` (encoded as -`.nil` on the wire). My SDK was passing `statusSize=0` + `status=[]` (empty -byte array). Updated SDK to send the native-matching values. - -Investigation step 2 — verified DelT still doesn't work standalone: with -the ref-input fix, DelT now returns `false` (not `true`-and-no-effect). -Tag continues to persist in `Runtime.dbo.Tag`. So the wire-byte parity -fix moved the symptom but didn't resolve the root cause. - -Investigation step 3 — discovered EnsureTagAsync is **also** silently -broken: byte-for-byte wire matches captured native EnsT2 (golden test -passes), but the call returns false and does NOT create the tag in the -DB. The earlier "EnsureTagAsync round-trip test passing" was relying on -the persistent tag from the broken DelT — a false positive. - -Two distinct issues remain: -- EnsT2 silently fails server-side (returns false; no tag created) -- DelT returns false even with native-matching wire bytes; needs deeper - investigation (likely the SDK's WCF channel state vs the native - HistorianAccess instance state) - -Diagnostic tooling for next session: write a custom -`IClientMessageInspector` for the SDK's WCF channel that captures -outgoing DelT bytes to a file. Compare byte-for-byte against the -captured native DelT (offset by offset, not just per-field) to isolate -the difference. - -## Phase 2 remaining work (revised — narrower scope) - -1. Decode the 146-byte EnsT2(Float) CTagMetadata against the IL of - `CTagUtil.ConvertTagMetadataToHistorianTag` (token `0x060055CE`), - then implement `HistorianAddTagsProtocol.SerializeAnalogCTagMetadata`. - Same approach for discrete/string variants — capture each by passing - `--write-data-type Discrete` / `String` to the harness. -2. Capture DelT wire bytes by running the harness with - `--write-delete-after`. -3. Implement public `EnsureTagAsync` + `DeleteTagAsync` only. **Drop - `WriteValueAsync` from this plan.** -4. (Stretch) probe `AddRevisionValuesBegin/Value/End` against a tag that - IS in the server cache (e.g., SysTimeSec) to see whether the revision - path bypasses the cache check. - -`WriteValueAsync` is now an OPEN QUESTION: is the only viable path for -client-driven writes the AVEVA REST API or the Application Server SDK? -File a separate plan for that investigation if SDK consumers actually -need data-write support. - -## Phase 1 findings (recorded here, not implementing) - -### §3.4 ModifyData/DeleteData — ELIMINATED FROM SCOPE +#### §3.4 ModifyData/DeleteData — ELIMINATED FROM SCOPE `methods aahClientManaged.dll` returns no managed wrapper for any of: `EditValue`, `ModifyValue`, `EditData`, `DeleteData`, `ModifyData`, `OverwriteData`. Per the plan's §3.4 disposition rule, this op is REST-only / SMC-only and remains out of scope for the SDK. -### §4.a Native serializers identified (token IDs for future Phase 2) +#### §4.a Native serializers identified The wrapper does have managed-public write API: @@ -246,494 +138,91 @@ The wrapper does have managed-public write API: | `ArchestrA.HistorianAccess.AddNonStreamedValue` | `0x0600618F/90` (2 overloads) | Pushes one timestamped value, non-stream mode | | `ArchestrA.HistorianAccess.DeleteTags` | `0x060061A4` | Removes tags (drives `DelT`) | | `ArchestrA.HistorianAccess.AddVersionedStreamedValue` | `0x0600616F` | Pushes versioned value (rev edit) | -| `ArchestrA.HistorianAccess.AddRevisionValuesBegin/Value/End/AddRevisionValues` | `0x06006175-77, 0x0600617F` | Multi-row revision write (replaces `ModifyData` use case) | +| `ArchestrA.HistorianAccess.AddRevisionValuesBegin/Value/End/AddRevisionValues` | `0x06006175-77, 0x0600617F` | Multi-row revision write | -So even though the engine doesn't expose `ModifyData` over WCF, the -**revision-write path** (`AddRevisionValuesBegin → AddRevisionValue * N → -AddRevisionValuesEnd`) covers the bulk-modify use case. This is a NEW -discovery worth folding into the Phase 2 scope. +Native serializer for `EnsT2`: +`.CTagUtil.ConvertTagMetadataToHistorianTag` at token +`0x060055CE`. WCF wrapper for `AddS2`: +`.CHistoryConnectionWCF.AddStreamValuesToHistorian` at token +`0x0600404C`. -Native serializer for `EnsT2(analog/discrete/string)`: -**`.CTagUtil.ConvertTagMetadataToHistorianTag`** at token -`0x060055CE` (412 IL instructions, 10 locals). Calls `CTagMetadata.GetUnit`, -`GetMessage0`, `GetMessage1`, `GetMaxLength`, `GetMinRaw`, `GetMaxRaw`, -`GetMinEU`, `GetMaxEU`, `GetIntegralDivisor`, `GetDefaultTagRate`, -`GetRolloverValue`, plus `CDataType.IsAnalog/IsWideString/GetRawType/ -GetTagType` — i.e. every field the analog `CTagMetadata` shape would -need is wired through this method. Decoding it line-by-line **OR** -capturing live wire bytes against a sandbox tag are the two ways -forward. +### Phase 2 follow-on findings (2026-05-04, second pass) -WCF wrapper for `AddS2`: **`.CHistoryConnectionWCF.AddStreamValuesToHistorian`** -at token `0x0600404C`. Confirms the on-wire shape is -`IHistoryServiceContract2.AddStreamValues2(string handle, byte[] pBuf, -out byte[] errorBuffer)` — matches our existing contract. Handle is -the same Open2 v6 session GUID we already extract. +**The AddS2 prereq is architectural, not protocol-level.** Three +follow-up attempts to trigger AddS2 from the sandbox harness all hit +a client-side gate before any AddS2 byte reaches the wire: -### Phase 2 chicken-and-egg resolved +1. TagKey synthetic→real override: even with the real `wwTagKey` the + server returns error 129 "Tag not found in cache". +2. Server-cache settle wait of 8s: error 129 persists. +3. Fresh process / fresh connection (skip AddTag): error 129; no AddS2 + bytes sent on wire. -Per §5 ordering: §3.1 (EnsT2) must come before §3.2 (AddS2) because -AddS2 needs an existing tag. The sandbox tag itself is created BY -the first §3.1 EnsT2 test. So the very first write-flow run creates -`RetestSdkWriteSandbox`. No SMC required — the chain is closed. +The Historian engine's runtime tag cache only ingests tags from +configured IOServers / Application Server pipelines, not from +`HistorianAccess.AddTag`-only flows. `WriteValueAsync` cannot be +implemented as a generic client API against this server architecture. -### Open question (was §8.6) answered +### Phase 2 results (write captures + EnsureTagAsync/DeleteTagAsync) -The wrapper exposes `AddStreamedValue` AND `AddNonStreamedValue`. -The latter is the documented path for backfilling values older than -`RealTimeWindow`. So the SDK should expose both modes, not just -`AddStreamedValue`. Update the success criteria for `AddS2` -accordingly. - -### Phase 2 next steps (NOT EXECUTED in this session) - -1. Extend `tools/AVEVA.Historian.NativeTraceHarness/Program.cs` with a - `--scenario write` that calls `HistorianAccess.AddTag` (creating - `RetestSdkWriteSandbox` if absent) followed by - `HistorianAccess.AddStreamedValue`. New args: - `--write-sandbox-tag ` (default: `RetestSdkWriteSandbox`), - `--write-value `, `--write-data-type analog|discrete|string`. -2. Run the harness with `instrument-wcf-writemessage` + - `instrument-wcf-readmessage` instrumented copies of - `aahClientManaged.dll` to capture the full write flow. -3. Decode `EnsT2(analog)` `InBuff` bytes against the IL of - `CTagUtil.ConvertTagMetadataToHistorianTag` (token `0x060055CE`). -4. Decode `AddS2` `pBuf` bytes against the IL of - `CHistoryConnectionWCF.AddStreamValuesToHistorian` (token - `0x0600404C`). -5. Implement `WriteValueAsync`, `EnsureTagAsync`, `DeleteTagAsync` per - §4.e of the original plan; live tests gated by - `HISTORIAN_WRITE_SANDBOX_TAG`. - -Phase 2 was deferred because (a) it requires extending the harness -(non-trivial scaffolding) and (b) per safety §1, even sandbox-tag -writes warrant explicit operator approval before the first run. The -operator decides whether to proceed; if yes, the instructions above -are executable as-is. - ---- - -Original plan content below. - - - -## 1. Goal - -"Write commands work" means the production SDK at -`src/AVEVA.Historian.Client/` performs these operations end-to-end -against a live AVEVA Historian, with parsed responses, golden-byte -unit tests, and gated live integration tests. - -In scope: - -1. **`AddS2` (`IHistoryServiceContract2.AddStreamValues2`)** — push - one or more timestamped samples for an existing historized tag. - Primary use case: an OPC UA driver pushing values to the - Historian. -2. **`EnsT2` (`IHistoryServiceContract2.EnsureTags2`) for - analog/discrete/string data tags** — partially decoded for the - `CM_EVENT` AnE-event tag in - `src/AVEVA.Historian.Client/Wcf/HistorianAddTagsProtocol.cs`. The - `CTagMetadata` byte layout for `CDataType` ∈ {1, 2, 3, 4} is the - new evidence target. -3. **`DelT` (`IHistoryServiceContract2.DeleteTags`)** — needed for - safe sandbox cleanup during RE. -4. **`ModifyData` / `DeleteData`** — only if §3.4 method discovery - confirms a managed WCF op exists. - -Out of scope: tag-extended-properties (`AddTEx` / `DelTep`), -`ExKey`, `SetSFP`, snapshot send (`SendSnapshotBegin/End/Snapshot`), -tag-id-pair maintenance, shard splits, flush ops, all -`IStorageServiceContract` writes (engine-internal — see §6.d), event -writes (events come from AVEVA AnE, we only read them), schema -changes (forbidden over the wire). - -## 2. Safety Constraints - -The Runtime DB is production data even on `localhost`. `AddS2` -writes are persistent — they go to compressed history blocks and -cannot be removed through any client-facing surface. - -Hard rules: - -1. **Single dedicated sandbox tag.** Add env var - `HISTORIAN_WRITE_SANDBOX_TAG = "RetestSdkWriteSandbox"`. Live - write tests refuse to run when unset, even when other - `HISTORIAN_*` vars are set. -2. **Never write to** any tag named in `HISTORIAN_TEST_TAG`, - `HISTORIAN_TAG_FILTER`, the docs, the test fixtures, or the - captured RE ndjson. The read fixture - `OtOpcUaParityTest_001.Counter` is OFF-LIMITS for writes. -3. **Documented rollback.** Every write session records its time - window to - `artifacts/reverse-engineering/write-sandbox-window-.json` - so SQL `SELECT * FROM History WHERE wwTagKey = ? AND DateTime - BETWEEN @s AND @e` can identify exactly which rows the session - inserted. Tag rollback is via decoded `DelT` (§3.3) once - available, or manually via System Management Console until then. -4. **Time bounds on writes.** Every `AddS2` test uses - `DateTime.UtcNow` ± a small offset, so writes always land inside - the live `RealTimeWindow` / `FutureTimeThreshold` system - parameters and cannot accidentally overwrite older blocks. -5. **No customer / corporate hosts.** `localhost` only. -6. **Sanitization scan after every session:** - `rg -n "(?i)(password|credential|secret|token|||)" docs\reverse-engineering scripts tools docs\plans`. - -Soft rules: - -- Use a separate captures dir - (`artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/`) - so write captures don't contaminate the existing read/event - ndjson. -- New integration tests follow the existing gating pattern in - `tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs` - (`Skip = ...` when env var unset). - -## 3. Discovery Workstreams - -### 3.1 EnsT2 for analog/discrete/string tags (priority 1) - -- WCF op: `aa/Hist/EnsT2`. -- Contract: - `src/AVEVA.Historian.Client/Wcf/Contracts/IHistoryServiceContract2.cs:82-89`, - already declared with `[MessageParameter(Name = "InBuff" / "OutBuff")]`. -- Existing code: `HistorianAddTagsProtocol.SerializeCmEventCTagMetadata` - builds the `CDataType=5` (event) shape. -- Missing: the `CTagMetadata` byte layout for `CDataType ∈ {1, 2, - 3, 4}` (analog double, discrete, string, analog int per the - type-code table in `data-query-request-ctor-il-latest.txt`); - whether the optional-mask `0x0086` and the 5-byte trailer - `2F 27 01 01 01` change per type; analog engineering-units / range - / deadband fields (likely populate the bytes that are zero in the - event-tag fixture). - -### 3.2 AddS2 stream values (priority 1) - -- WCF op: `aa/Hist/AddS2`. -- Contract: - `src/AVEVA.Historian.Client/Wcf/Contracts/IHistoryServiceContract2.cs:75-80`, - already has `[MessageParameter(Name = "pBuf")]`. **Audit - requirement:** verify against `ildasm aahClientAccessPoint.exe` - that `Handle` and `errorBuffer` parameter names also match — the - handoff's parameter-name-mismatch class has bitten ~30 ops. -- Missing: entire `pBuf` byte layout (likely `UInt16 version + UInt32 - sampleCount + N × {tagId GUID, FILETIME, qualityByte, value typed - by CDataType}`); whether `Handle` is the same Open2 v6 session GUID - as `UpdC3`/`RTag2`/`EnsT2`; the auth-chain prereqs (event flow - needed Stat priming + Trx/Stat/Retr `GetV` between RTag2 and EnsT2; - writes may have a different chain); success vs error response - shape. - -### 3.3 DelT tag deletion (priority 2 — needed for safe RE) - -- WCF op: `aa/Hist/DelT`. -- Contract: - `src/AVEVA.Historian.Client/Wcf/Contracts/IHistoryServiceContract2.cs:21-30`. -- Missing: `tagNames` byte layout (likely length-prefixed - compact-ASCII per the handoff convention); whether server refuses - to delete tags with stored history or cascades; whether `DelT` is - sufficient to fully unregister or leaves orphan rows in - `Runtime.dbo.Tag`. - -### 3.4 ModifyData / DeleteData (priority 3 — exists?) - -No corresponding WCF op is currently declared. **First step:** static -inspection to confirm any managed wrapper exists. - -```powershell -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll EditValue -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll ModifyValue -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll EditData -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll DeleteData -``` - -If no managed wrapper exists, this op is REST-only / SMC-only — -mark as **out of scope** in this doc. Otherwise decode like -§3.1/§3.2. - -Parallelism: 3.1 and 3.3 can be developed in parallel because the -operator can create the sandbox tag manually via SMC while SDK code -is being written. 3.2 cannot meaningfully proceed until 3.1 (or the -manual tag) exists. 3.4 method discovery is cheap and may eliminate -its own scope. - -## 4. RE Steps in Execution Order - -For each workstream above, run these five steps. Mirrors the read -+ event flows that recovered the existing protocol. - -### 4.a Static method discovery - -Find the native serializer: - -```powershell -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll AddS -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll EnsureTag -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- methods current\aahClientManaged.dll DeleteTag -``` - -Dump IL for each method of interest: - -```powershell -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- dnlib-method --instructions current\aahClientManaged.dll -``` - -Save sanitized excerpts to -`docs/reverse-engineering/dnlib--il-latest.txt`. - -### 4.b Wire-byte capture for the request - -Same IL-rewrite tooling that captured the 27 outgoing event calls: - -```powershell -$captureDir = "artifacts\reverse-engineering\instrumented-wcf-writemessage-writes" -New-Item -ItemType Directory -Force -Path $captureDir | Out-Null -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- instrument-wcf-writemessage current\aahClientManaged.dll "$captureDir\aahClientManaged.dll" -Copy-Item -Force "$captureDir\aahClientManaged.dll" "$captureDir\current-copy\aahClientManaged.dll" -$env:AVEVA_HISTORIAN_RE_CAPTURE = (Resolve-Path $captureDir).Path + "\writemessage-capture-write-latest.ndjson" -``` - -A new harness scenario `--scenario write` needs to be added to -`tools/AVEVA.Historian.NativeTraceHarness` to drive the native -wrapper's `AddStreamValues2` against the sandbox tag. Suggested -new args: `--write-sandbox-tag`, `--write-value`. - -### 4.c Wire-byte capture for the response - -Symmetric `instrument-wcf-readmessage`: - -```powershell -dotnet run --no-build --project tools\AVEVA.Historian.ReverseEngineering -- instrument-wcf-readmessage current\aahClientManaged.dll "$captureDir\aahClientManaged.dll" -``` - -The success response for `AddS2` is just `true` + -empty `errorBuffer`. **Capture at least one negative case** (write -to non-existent tag, or write with malformed CDataType) so the -orchestrator can surface diagnostics like -`HistorianWcfEventOrchestrator.LastErrorBufferDescription`. - -### 4.d Decode against IL - -Strip SOAP/MDAS envelope; align byte offsets against the native -serializer IL from 4.a (the `ldc.i4 / call WriteByte` sequence -makes field order and constants explicit); cross-reference the -`CDataType` table from `data-query-request-ctor-il-latest.txt` to -interpret typed value bytes; write a parser-and-builder pair and -verify against the captured bytes before committing. - -### 4.e Implement managed serializer + tests - -New code under `src/AVEVA.Historian.Client/Wcf/`: - -- `HistorianAddStreamValuesProtocol.cs` — `Serialize(...)` returns - `byte[] pBuf`, mirroring `HistorianAddTagsProtocol`. -- Extend (or split) `HistorianAddTagsProtocol` for the analog / - discrete / string `EnsT2` shapes. -- `HistorianWcfWriteOrchestrator.cs` — chains `Hist.GetV → - Hist.ValCl × 2 → Hist.Open2 → UpdC3 → priming chain (TBD per - §3.2) → AddS2 loop → Close2`. - -Public surface on `HistorianClient`: - -- `WriteValueAsync(tag, value, timestampUtc, quality)` -- `WriteValuesAsync(IReadOnlyList)` -- `EnsureTagAsync(HistorianTagDefinition)` -- `DeleteTagAsync(string tagName)` - -Until evidence supports each path, throw -`ProtocolEvidenceMissingException` (mirrors the existing read -guardrail). - -Unit tests under `tests/AVEVA.Historian.Client.Tests/Wcf/`: - -- `WcfAddStreamValuesProtocolTests` — golden-byte tests for one - analog, one discrete, one string write. -- `WcfEnsureTagsProtocolTests` — golden-byte tests for the - analog/discrete/string `CTagMetadata` shapes. -- Extend `ProtocolGuardrailTests` so any not-yet-implemented write - path still throws `ProtocolEvidenceMissingException`. - -Live integration tests in `HistorianClientIntegrationTests.cs`, -gated on `HISTORIAN_WRITE_SANDBOX_TAG`: -`WriteValueAsync_WithinDocumentedWindow_PersistsToHistorianDb` -writes a unique value, reads it back via `ReadRawAsync`, and -verifies via direct `sqlcmd` to the History extension table. - -## 5. Order of Operations +EnsT2 + DelT priming chain captured (no `RTag2` between Open2 and +EnsT2): ``` -3.4 method discovery (cheap; may eliminate scope) - │ - ▼ -3.1 EnsT2 (analog/discrete/string) ──► sandbox tag exists - │ - ├─────────────────────────────┐ - ▼ ▼ -3.2 AddS2 (priority 1) 3.3 DelT (sandbox cleanup) - │ - ▼ -3.4 ModifyData/DeleteData (only if 3.4 confirmed scope) - │ - ▼ -public surface, golden-byte tests, integration tests +Hist.GetV → Hist.GetI ×2 → Hist.ValCl ×2 → Hist.Open2 → +Stat.GetV ×2 → Stat.GETHI ×2 → Hist.UpdC3 → +Stat.GetSystemParameter ×7 → Trx.GetV → Stat.GetV → Retr.GetV → +Hist.EnsT2 → Hist.Close2 ``` -3.2 is the headline win and depends only on 3.1 (or a manually -created sandbox tag). 3.3 must land before any commit that -programmatically creates new tags; until then, manual SMC deletion -is the documented rollback. +Open2 with `NativeIntegratedWriteEnabledConnectionMode = 0x401` +(Process | Write | IntegratedSecurity) is required — read-mode +`0x402` makes the server return err 132 `OperationNotEnabled` +silently. The analog Float `CTagMetadata` payload is 144 bytes with a +leading `0x4E` marker byte and a 2-byte trailer `FE xx` where the +second byte is the ApplyScaling flag (`00` = false, `01` = true). -## 6. Risks and Mitigations +### ApplyScaling resolution (2026-05-04) -### 6.a Auth chain may differ for writes +Earlier docs claimed "MinRaw is mirrored to MinEU — server quirk, +not SDK bug". That conclusion was based on tests that always set +ApplyScaling=false on the native side. Re-running with +`set_ApplyScaling(true)` on the harness and capturing wire bytes for +both values revealed: -Reads use `Hist.Open2(ConnectionMode = 0x402)`. Events use the same -`0x402` plus a Stat-priming chain. Writes may need a different -mode (the handoff notes `0x501` was an unverified guess for -events; writes may legitimately need `0x401` or another value). +- ApplyScaling=false → trailer = `FE 00` → server mirrors MinRaw→MinEU, + sets `AnalogTag.Scaling=0` +- ApplyScaling=true → trailer = `FE 01` → server persists distinct + MinRaw/MaxRaw, sets `AnalogTag.Scaling=1` -Mitigation: capture the *full* WriteMessage sequence for a native -write session (not just `AddS2`) to see what `Open2` payload and -priming calls the native wrapper sends. +The `IHistoryServiceContract2` surface has **no `UpdateTags` +operation**. Distinct MinRaw/MaxRaw persistence is achieved entirely +by toggling that one byte in the EnsT2 payload. The SDK now exposes +this via `HistorianTagDefinition.ApplyScaling`. -### 6.b Server-side session-table requirement +Capture artifacts: +`artifacts/reverse-engineering/apply-scaling-experiment/enst2-applyscaling-{false,true}.ndjson`. -Writes may require `RTag2` after `EnsT2` and before `AddS2` (the -event flow needs `RTag2(CmEventTagId)`). The "tag identifier" the -server returns from `EnsT2` may differ from the GUID the client -seeded. +### Original goal section (preserved for historical reference) -Mitigation: capture the analog `EnsT2` `OutBuff` (event flow's was -a 45-byte echo) and verify whether subsequent `AddS2` payloads -reference the client-seeded GUID, the server-returned GUID, or a -numeric `wwTagKey`. SQL ground truth: `SELECT TagName, wwTagKey -FROM Tag WHERE TagName = '...'`. +"Write commands work" was originally defined as the four ops: +`EnsT2`, `AddS2`, `DelT`, and `ModifyData/DeleteData`. The realized +scope is `EnsT2 + DelT` only. AddS2 is permanently blocked by +server architecture; ModifyData/DeleteData were eliminated by +static analysis (no managed wrapper exists). The +`AddRevisionValuesBegin/Value/End` chain remains a stretch goal +(item D2 in the current plan) — it was never investigated because +no SDK consumer has asked for revision writes. -### 6.c Silent-success failure mode +### Original safety rules (still applicable) -`AddS2` may return `true` but no row appears in the History -extension table — the engine silently drops samples outside the -`FutureTimeThreshold` / `RealTimeWindow` system parameters (which -the event flow now reads). - -Mitigation: always write at `DateTime.UtcNow`; cross-check with -SQL after every test: - -```sql -SELECT TOP 5 DateTime, Value, QualityDetail -FROM History -WHERE wwTagKey = (SELECT wwTagKey FROM Tag WHERE TagName = @sandbox) - AND DateTime BETWEEN @windowStart AND @windowEnd -ORDER BY DateTime DESC; -``` - -Surface `FutureTimeThreshold` / `RealTimeWindow` via existing -`GetSystemParameterAsync` so failures are diagnosable. - -### 6.d Storage service vs History service - -`IStorageServiceContract` also exposes `AddT/AddS/AddS2/DelT`. The -working hypothesis is that `/Hist` is client-facing and `/Stor` is -engine-internal, but it's not yet verified. - -Mitigation: the WriteMessage capture (§4.b) shows the actual -service path on the wire. If it goes to `/Stor`, update the -orchestrator. Do NOT preemptively implement against both. - -### 6.e Parameter-name mismatches - -Handoff already flagged `EnsT`, `EnsT2`, `RTag2`, `ExKey`, `StJb`, -`GtJb` for the same `inBuff`/`inputBuffer` mismatch class that -broke reads for weeks. Until each is audited against the server -contract, requests bind to null and the server NREs. - -Mitigation: before the first write WriteMessage capture, run an -`ildasm` audit against `aahClientAccessPoint.exe` for the exact -parameter names of `EnsT2`, `AddS2`, and `DelT`, and reconcile -against the existing `[MessageParameter]` attributes. - -### 6.f Customer-data exposure in capture files - -Write captures contain the sandbox tag name and any value the test -wrote. Not secrets, but noise. - -Mitigation: keep all -`instrumented-wcf-writemessage-writes/` artifacts under -`artifacts/` (already gitignored). Sanitize tag names to -`` before committing decoded bytes into -`docs/reverse-engineering/`. - -## 7. Success Criteria - -Per op: - -- **`EnsT2(analog)`**: `EnsureTagAsync(new HistorianTagDefinition { - Name = sandbox, DataType = Analog })` returns success; - `sqlcmd -E -S . -d Runtime -Q "SELECT TagName FROM Tag WHERE - TagName = '...'"` returns one row. -- **`EnsT2(discrete, string)`**: same shape with corresponding - `DataType`; SQL check uses `DiscreteTag` / `StringTag` view. -- **`AddS2`**: `WriteValueAsync(sandbox, 42.0, DateTime.UtcNow)` - returns success; `ReadRawAsync` returns the value; - `SELECT TOP 1 Value FROM History WHERE wwTagKey = ? AND DateTime - BETWEEN ? AND ?` returns the same value. -- **`DelT`**: `DeleteTagAsync(sandbox)` returns success and SQL - returns zero rows from `Tag`. -- **`ModifyData` / `DeleteData`**: deferred until §3.4 method - discovery confirms scope. - -Cross-cutting: - -- All new code in `src/AVEVA.Historian.Client/` is pure managed - .NET 10. No new P/Invoke beyond the existing `HistorianSspiClient`. -- Every new op has a golden-byte unit test. -- `dotnet test .\Histsdk.slnx --no-build --logger - "console;verbosity=minimal"` passes 100%. -- With `HISTORIAN_HOST=localhost`, - `HISTORIAN_WRITE_SANDBOX_TAG=RetestSdkWriteSandbox` set, write - integration tests pass and leave zero residue (test `Dispose` - calls `DelT` for cleanup). -- Sanitization scan returns no real secrets. -- `CLAUDE.md` "Required SDK Surface" updated to add the new write - ops — this is a SCOPE CHANGE that must land *alongside* the - evidence, not before. Do not update the SDK surface doc until - 3.1 + 3.2 are at least live-test-green. - -## 8. Open Questions - -1. Does `AddS2` go through `/Hist` or `/Stor` on the wire? -2. Does the sandbox tag need pre-configuration via System - Management Console once before `EnsT2` will accept it from a - client (e.g. for `Storage` / `wwDomain` rows the wire protocol - may not be able to populate)? -3. What `ConnectionMode` does the native wrapper use for write - sessions — `0x402` (read mode reused), `0x401`, or something - else? -4. Does `EnsT2(analog)` require any optional Archestra - engineering-units fields, or are they purely cosmetic? Affects - how minimal `HistorianTagDefinition` can be. -5. Server-side throttles on writes (max samples per AddS2, max - calls per second) — need to surface as batching guidance? -6. What does the server return when `AddS2` is called with a - timestamp older than the tag's earliest stored block? Some - historians silently drop, some error, some accept-and-overwrite. -7. Does the SDK expose write quality as the same - `HistorianSample.Quality` enum used on reads, or a smaller - subset (good/bad)? -8. Is there a managed-side `DelT` path at all? If - `aahClientManaged` only exposes deletion via SMC, §3.3 is - "manual SMC only" and must be documented as such. - -## 9. Docs To Update Once Each Workstream Lands - -- `CLAUDE.md` "Required SDK Surface" — add `WriteValueAsync`, - `EnsureTagAsync`, `DeleteTagAsync` once 3.1+3.2+3.3 land. -- `AGENTS.md` "Required SDK Surface" — same; update the "alarm-event - write path is dormant" note. -- `docs/reverse-engineering/handoff.md` — add a "Write-flow prereqs" - section symmetric to the existing "Event-flow prereqs". -- `docs/reverse-engineering/wcf-contract-evidence.md` — add evidence - rows for `EnsT2(analog/discrete/string)`, `AddS2`, `DelT`. -- `docs/reverse-engineering/implementation-status.md` — flip - status from "out of scope" to "implemented". -- `README.md` — operation status table. +- Single dedicated sandbox tag per RE session, name must start with + `RetestSdkWrite`. +- Never write to any tag named in `HISTORIAN_TEST_TAG`, + `HISTORIAN_TAG_FILTER`, the docs, or the captured RE ndjson. +- Time bounds on writes: every test uses `DateTime.UtcNow` so writes + land inside the live `RealTimeWindow` / `FutureTimeThreshold`. +- `localhost` only; no customer / corporate hosts. +- Sanitization scan after every session. +- Write captures live in `artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/` + (gitignored). diff --git a/docs/reverse-engineering/handoff.md b/docs/reverse-engineering/handoff.md index faef8eb..a195a27 100644 --- a/docs/reverse-engineering/handoff.md +++ b/docs/reverse-engineering/handoff.md @@ -1,6 +1,6 @@ # AVEVA Historian Managed Driver Handoff -Last updated: 2026-05-04 (event-flow prereqs) +Last updated: 2026-05-04 (write surface live: EnsT2 + DelT + ApplyScaling) ## Project Direction @@ -12,7 +12,7 @@ Do not pivot to REST or a P/Invoke production shim unless the project requirements change. Native and P/Invoke tools in this repo are reverse engineering aids only. -Required production surface remains narrowly scoped: +Required production surface (all live-verified): - `ProbeAsync` - `ReadRawAsync` @@ -21,8 +21,23 @@ Required production surface remains narrowly scoped: - `ReadEventsAsync` - `BrowseTagNamesAsync` - `GetTagMetadataAsync` +- Status helpers: `GetConnectionStatusAsync`, `GetStoreForwardStatusAsync`, `GetSystemParameterAsync` -Writes are out of scope for the current pass. +Write surface (added 2026-05-04 by explicit user request — see +`docs/plans/write-commands-reverse-engineering.md` Status section): + +- `EnsureTagAsync` for analog Float / Double / Int2 / Int4 / UInt4 + (with optional `ApplyScaling=true` for distinct MinRaw / MaxRaw + persistence — server sets `AnalogTag.Scaling=1` when the EnsT2 + trailer's second byte is `0x01` instead of `0x00`). +- `DeleteTagAsync`. + +`AddS2` (write samples) is **architecturally blocked** — server +runtime cache only ingests from configured IOServers / Application +Server pipelines. Discrete / String / Int1 / Int8 / UInt8 EnsT2 fail +at native `AddTag` and are unsupported. There is no `UpdateTags` +operation on the WCF surface; the misnomer in earlier write-up +drafts has been removed. ## Repository Map diff --git a/docs/reverse-engineering/implementation-status.md b/docs/reverse-engineering/implementation-status.md index 938440f..6057712 100644 --- a/docs/reverse-engineering/implementation-status.md +++ b/docs/reverse-engineering/implementation-status.md @@ -3,13 +3,23 @@ ## Completed - Production SDK targets `net10.0` and has no AVEVA binary references. -- Public API now includes the intended parity surface: +- Public API includes the full intended parity surface: - TCP probe - raw, aggregate, at-time, and block history reads - event reads - tag browse and metadata calls - connection, store-forward, and system-parameter status calls - - write-back intentionally remains out of scope for this read-only SDK pass + - **`EnsureTagAsync`** for analog Float/Double/Int2/Int4/UInt4 with + optional `ApplyScaling=true` for distinct MinRaw/MaxRaw persistence + (live-verified end-to-end against `localhost`; SQL post-check confirms + `AnalogTag.Scaling=1` and distinct raw bounds when the flag is set) + - **`DeleteTagAsync`** (live-verified) + - **AddS2 (write samples) is architecturally blocked** — server runtime + cache only ingests from configured IOServers / Application Server + pipelines, not from `HistorianAccess.AddTag`-only flows. Three + independent reproduction attempts confirmed the same + `129 "Tag not found in cache"` failure even with the real wwTagKey, + fresh sessions, and 8s settle waits. Not a protocol gap. - Internal protocol scaffolding exists: - `HistorianConnection` - `HistorianFrameReader` diff --git a/docs/reverse-engineering/wcf-contract-evidence.md b/docs/reverse-engineering/wcf-contract-evidence.md index 63fa114..03c05da 100644 --- a/docs/reverse-engineering/wcf-contract-evidence.md +++ b/docs/reverse-engineering/wcf-contract-evidence.md @@ -159,5 +159,33 @@ the earlier speculative raw-frame layer. handle `0`. See `wcf-status-localhost.md`. - Query request and response byte-buffer layouts are still proprietary payloads inside WCF operations such as `StartQuery` and `GetNextQueryResultBuffer`. -- Write payload layouts remain out of scope until read/query payloads are - decoded and fixture-backed. +- Write payload layouts decoded for the two supported ops: + - `Hist.EnsT2(analog)` 144-byte `CTagMetadata` `InBuff` payload — + leading `0x4E` marker, fixed 10-byte signature, 1-byte CDataType + discriminator (`0x01` Float / `0x21` Double / `0x09` UInt2 / `0x11` + UInt4 / `0x29` Int2 / `0x31` Int4), 16 zero placeholder bytes, + compact-ASCII tag name, 16 bytes of `0xFF`, compact-ASCII description, + compact-ASCII `MDAS`, 7-byte flag block, uint32 storage rate, + int64 FILETIME, scaling block (compact `1A 03` for default + 0/100/0/100 ranges OR `1F 00` + 4 doubles MinEU/MaxEU/MinRaw/MaxRaw + for explicit), compact-ASCII engineering unit, uint32 `0x2710` + constant, double 1.0 (IntegralDivisor), 2-byte trailer `FE xx` + where `xx` is the ApplyScaling flag (`0x00` false / `0x01` true). + Live-verified: with `0x01` the server persists distinct + MinRaw/MaxRaw and sets `AnalogTag.Scaling=1`; with `0x00` it + mirrors MinRaw to MinEU. Captured fixtures live at + `artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/` + (default ranges) and + `artifacts/reverse-engineering/apply-scaling-experiment/` (both + ApplyScaling values for the same input ranges). Connection mode is + `0x401` (Process | Write | IntegratedSecurity) — the read-mode + `0x402` makes the server return err 132 silently. + - `Hist.DelT` `tagNames` byte buffer — `ushort 0x6751`, `ushort 1`, + `uint32 tagCount`, then per tag `uint32 charCount + UTF-16-LE chars`. + Decoded via wire capture against the sandbox tag. + - `Hist.AddS2` (write samples) is architecturally blocked — server + runtime cache requires IOServer / Application Server pipeline + registration, not just a `Tag` row in `Runtime.dbo`. Three + reproduction attempts (real wwTagKey, fresh session, 8s settle + wait) confirmed `129 "Tag not found in cache"` is the gate. No + AddS2 wire bytes leave the client. diff --git a/src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs b/src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs index 9e6f14c..65e42b5 100644 --- a/src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs +++ b/src/AVEVA.Historian.Client/Models/HistorianTagDefinition.cs @@ -6,6 +6,11 @@ namespace AVEVA.Historian.Client.Models; /// String/Int1/Int8/UInt8 types failed at native AddTag — likely require a different /// path and are intentionally not supported. MinEU/MaxEU/MinRaw/MaxRaw are now encoded /// into the wire payload (see HistorianTagWriteProtocol). +/// +/// Semantics: EnsureTagAsync is an upsert. Calling it twice on the same +/// with different fields succeeds both times; the second call +/// updates Description, MinEU, MaxEU, MinRaw, MaxRaw, and AnalogTag.Scaling on the +/// existing row (verified 2026-05-04 by direct SQL inspection after sequential calls). /// public sealed record HistorianTagDefinition { @@ -28,18 +33,32 @@ public sealed record HistorianTagDefinition public double MaxEU { get; init; } = 100.0; /// - /// Raw lower bound (pre-scaling). Default 0. Note: with ApplyScaling=false (the - /// only path the SDK currently exposes), the server appears to mirror MinRaw to - /// MinEU on EnsureTags2 — verified 2026-05-04 against both native and managed - /// clients with the same input. The value is sent on the wire but not persisted - /// independently. To set distinct raw bounds, ApplyScaling=true plus a follow-up - /// UpdateTags call would be required (not yet wired). + /// Raw lower bound (pre-scaling). Default 0. Persisted distinctly only when + /// is true; with ApplyScaling=false the server mirrors + /// this to MinEU on EnsureTags2 (verified 2026-05-04 against both native and + /// managed clients). /// public double MinRaw { get; init; } /// /// Raw upper bound (pre-scaling). Default 100. See for the - /// server-side mirror caveat with ApplyScaling=false. + /// ApplyScaling caveat. /// public double MaxRaw { get; init; } = 100.0; + + /// + /// When true, the server persists / as + /// distinct values from / and sets + /// AnalogTag.Scaling = 1. When false (default), the server mirrors MinRaw + /// to MinEU and MaxRaw to MaxEU and sets AnalogTag.Scaling = 0. + /// + public bool ApplyScaling { get; init; } + + /// + /// Storage rate in milliseconds. Default 1000ms. The server only accepts + /// quantized values (observed valid set: 1000, 5000, 10000, 60000, 300000) — + /// non-quantized values cause to + /// return false. + /// + public uint StorageRateMs { get; init; } = 1000u; } diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianTagWriteProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianTagWriteProtocol.cs index 530862e..4d42b0d 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianTagWriteProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianTagWriteProtocol.cs @@ -26,13 +26,12 @@ namespace AVEVA.Historian.Client.Wcf; /// compact ASCII engineering unit /// uint32 = 0x2710 (10000 — purpose unclear; observed constant) /// 8-byte double = 1.0 (likely IntegralDivisor) -/// 2-byte trailer = FE 00 +/// 2-byte trailer = `FE 00` for ApplyScaling=false; `FE 01` for ApplyScaling=true /// -/// MinEU/MaxEU/MinRaw/MaxRaw fields and their wire positions are NOT yet decoded -/// from the captured fixture (the test tag used the defaults). The serializer accepts -/// those parameters from but their wire -/// representation is currently a TODO; for now they are not encoded into the -/// payload — the server uses defaults from the AnalogTag table after creation. +/// The trailer's second byte is the ApplyScaling flag — verified 2026-05-04 by +/// capturing native CTagMetadata bytes for both values with identical +/// MinEU/MaxEU/MinRaw/MaxRaw inputs and observing that the server persists distinct +/// MinRaw/MaxRaw (and sets AnalogTag.Scaling=1) only when this byte is 0x01. /// internal static class HistorianTagWriteProtocol { @@ -88,10 +87,14 @@ internal static class HistorianTagWriteProtocol private static readonly byte[] AnalogScalingDefaultsMarker = [0x1A, 0x03]; /// Explicit-scaling marker (2 bytes) — followed by 4 doubles in order MinEU, MaxEU, MinRaw, MaxRaw. private static readonly byte[] AnalogScalingExplicitMarker = [0x1F, 0x00]; - // Native trailer is 2 bytes; the prior 5-byte version included WCF EndElement - // closing markers (`01 01 01`) that the binary message encoder writes around the - // element — those are not part of the buffer content. - private static readonly byte[] AnalogTrailer = [0xFE, 0x00]; + // 2-byte trailer: `FE` marker + ApplyScaling byte (0x00 = false, 0x01 = true). Verified + // against native captures by toggling ApplyScaling on the harness and confirming that + // the server persists distinct MinRaw/MaxRaw + sets AnalogTag.Scaling=1 only when the + // second byte is 0x01. The WCF binary encoder may split InBuff across two + // Bytes8Text chunks (e.g., `9E B7 ... 9F 01 00`) which can make the trailer look + // 1-byte from the wire, but the semantic CTagMetadata content is always 2 bytes. + private static readonly byte[] AnalogTrailerScalingDisabled = [0xFE, 0x00]; + private static readonly byte[] AnalogTrailerScalingEnabled = [0xFE, 0x01]; private const double DefaultMinEU = 0.0; private const double DefaultMaxEU = 100.0; @@ -129,8 +132,13 @@ internal static class HistorianTagWriteProtocol double maxEU = DefaultMaxEU, double minRaw = DefaultMinRaw, double maxRaw = DefaultMaxRaw, - uint storageRateMs = DefaultStorageRateMs) + uint storageRateMs = DefaultStorageRateMs, + bool applyScaling = false) { + if (storageRateMs == 0) + { + throw new ArgumentOutOfRangeException(nameof(storageRateMs), "Storage rate must be > 0 ms."); + } ArgumentException.ThrowIfNullOrWhiteSpace(tagName); byte typeCode = GetAnalogDataTypeCode(dataType); @@ -164,7 +172,7 @@ internal static class HistorianTagWriteProtocol WriteCompactAscii(w, engineeringUnit ?? string.Empty); // var w.Write(IntegralDivisorMagic); // uint32 (purpose unclear — captured constant) w.Write(1.0); // double - w.Write(AnalogTrailer); // 2 bytes (FE 00) + w.Write(applyScaling ? AnalogTrailerScalingEnabled : AnalogTrailerScalingDisabled); return ms.ToArray(); } diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs index d8d6353..b559f14 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs @@ -108,7 +108,9 @@ internal sealed class HistorianWcfTagWriteOrchestrator minEU: definition.MinEU, maxEU: definition.MaxEU, minRaw: definition.MinRaw, - maxRaw: definition.MaxRaw); + maxRaw: definition.MaxRaw, + storageRateMs: definition.StorageRateMs, + applyScaling: definition.ApplyScaling); bool ok = historyChannel.EnsureTags2( handle: handle, diff --git a/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj b/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj index fc0a58f..172b102 100644 --- a/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj +++ b/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj @@ -10,6 +10,7 @@ + diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs index 0e82e0a..0604d1e 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs @@ -488,6 +488,179 @@ public sealed class HistorianClientIntegrationTests } } + [Fact] + public async Task EnsureTagAsync_NonDefaultStorageRate_PersistsToTagTable() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + + const string sandboxTag = "RetestSdkWriteStorageRateRT"; + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe, + }); + + try + { + bool ok = await client.EnsureTagAsync(new AVEVA.Historian.Client.Models.HistorianTagDefinition + { + TagName = sandboxTag, + Description = "SDK StorageRate round-trip", + EngineeringUnit = "test", + DataType = AVEVA.Historian.Client.Models.HistorianDataType.Float, + // Server only accepts quantized rates — 1000, 5000, 10000, 60000, 300000 ms. + StorageRateMs = 5000u, + }, CancellationToken.None); + Assert.True(ok, "EnsureTagAsync returned false"); + + using Microsoft.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;Encrypt=False;TrustServerCertificate=True"); + sql.Open(); + using Microsoft.Data.SqlClient.SqlCommand cmd = sql.CreateCommand(); + cmd.CommandText = "SELECT StorageRate FROM Tag WHERE TagName = @t"; + cmd.Parameters.AddWithValue("@t", sandboxTag); + object? rate = cmd.ExecuteScalar(); + Assert.NotNull(rate); + Assert.Equal(5000, Convert.ToInt32(rate)); + } + finally + { + await client.DeleteTagAsync(sandboxTag, CancellationToken.None); + } + } + + [Fact] + public async Task EnsureTagAsync_CalledTwiceOnSameTag_UpdatesFieldsInPlace() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + + const string sandboxTag = "RetestSdkWriteIdempotencyRT"; + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe, + }); + + try + { + bool firstOk = await client.EnsureTagAsync(new AVEVA.Historian.Client.Models.HistorianTagDefinition + { + TagName = sandboxTag, + Description = "First version", + EngineeringUnit = "test", + DataType = AVEVA.Historian.Client.Models.HistorianDataType.Float, + MinEU = 0.0, MaxEU = 100.0, MinRaw = 0.0, MaxRaw = 100.0, + ApplyScaling = false, + }, CancellationToken.None); + Assert.True(firstOk, "First EnsureTagAsync returned false"); + (string desc1, double minEU1, double maxEU1, double minRaw1, double maxRaw1, int scaling1) = ReadTagState(sandboxTag); + Assert.Equal("First version", desc1); + Assert.Equal(0.0, minEU1); + Assert.Equal(0, scaling1); + + bool secondOk = await client.EnsureTagAsync(new AVEVA.Historian.Client.Models.HistorianTagDefinition + { + TagName = sandboxTag, + Description = "Second version", + EngineeringUnit = "kPa", + DataType = AVEVA.Historian.Client.Models.HistorianDataType.Float, + MinEU = -50.0, MaxEU = 200.0, MinRaw = 10.0, MaxRaw = 4095.0, + ApplyScaling = true, + }, CancellationToken.None); + Assert.True(secondOk, "Second EnsureTagAsync returned false"); + (string desc2, double minEU2, double maxEU2, double minRaw2, double maxRaw2, int scaling2) = ReadTagState(sandboxTag); + + // EnsureTagAsync upserts: second call updates the existing row in place. + Assert.Equal("Second version", desc2); + Assert.Equal(-50.0, minEU2); + Assert.Equal(200.0, maxEU2); + Assert.Equal(10.0, minRaw2); + Assert.Equal(4095.0, maxRaw2); + Assert.Equal(1, scaling2); + } + finally + { + await client.DeleteTagAsync(sandboxTag, CancellationToken.None); + } + + static (string desc, double minEU, double maxEU, double minRaw, double maxRaw, int scaling) ReadTagState(string tagName) + { + using Microsoft.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;Encrypt=False;TrustServerCertificate=True"); + sql.Open(); + using Microsoft.Data.SqlClient.SqlCommand cmd = sql.CreateCommand(); + cmd.CommandText = "SELECT t.[Description], a.MinEU, a.MaxEU, a.MinRaw, a.MaxRaw, a.Scaling FROM Tag t JOIN AnalogTag a ON a.TagName=t.TagName WHERE t.TagName=@t"; + cmd.Parameters.AddWithValue("@t", tagName); + using Microsoft.Data.SqlClient.SqlDataReader r = cmd.ExecuteReader(); + Assert.True(r.Read(), $"Tag {tagName} not found"); + return (r.GetString(0), r.GetDouble(1), r.GetDouble(2), r.GetDouble(3), r.GetDouble(4), Convert.ToInt32(r.GetValue(5))); + } + } + + [Fact] + public async Task EnsureTagAsync_ApplyScalingTrue_PersistsDistinctMinRawAndMaxRaw() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + + const string sandboxTag = "RetestSdkWriteApplyScalingRT"; + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe, + }); + + AVEVA.Historian.Client.Models.HistorianTagDefinition definition = new() + { + TagName = sandboxTag, + Description = "SDK ApplyScaling round-trip", + EngineeringUnit = "test", + DataType = AVEVA.Historian.Client.Models.HistorianDataType.Float, + MinEU = -50.0, + MaxEU = 200.0, + MinRaw = 10.0, + MaxRaw = 4095.0, + ApplyScaling = true, + }; + + try + { + bool ensured = await client.EnsureTagAsync(definition, CancellationToken.None); + Assert.True(ensured, "EnsureTagAsync(ApplyScaling=true) returned false against the live Historian."); + + // Verify directly against the AnalogTag table — the read-path GetTagMetadataAsync + // surfaces only one of (MinRaw, MinEU); SQL is the unambiguous source of truth. + using Microsoft.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;Encrypt=False;TrustServerCertificate=True"); + sql.Open(); + using Microsoft.Data.SqlClient.SqlCommand cmd = sql.CreateCommand(); + cmd.CommandText = "SELECT MinEU, MaxEU, MinRaw, MaxRaw, Scaling FROM AnalogTag WHERE TagName = @t"; + cmd.Parameters.AddWithValue("@t", sandboxTag); + using Microsoft.Data.SqlClient.SqlDataReader r = cmd.ExecuteReader(); + Assert.True(r.Read(), $"AnalogTag row for {sandboxTag} not found after EnsureTag."); + Assert.Equal(-50.0, r.GetDouble(0)); + Assert.Equal(200.0, r.GetDouble(1)); + Assert.Equal(10.0, r.GetDouble(2)); + Assert.Equal(4095.0, r.GetDouble(3)); + Assert.Equal(1, Convert.ToInt32(r.GetValue(4))); + } + finally + { + await client.DeleteTagAsync(sandboxTag, CancellationToken.None); + } + } + [Fact] public async Task GetTagMetadataAsync_PopulatesDescriptionAndEuRangeForAnalogTag() { diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianTagWriteProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianTagWriteProtocolTests.cs index 3dce727..d057dfc 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianTagWriteProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianTagWriteProtocolTests.cs @@ -8,16 +8,9 @@ public sealed class HistorianTagWriteProtocolTests [Fact] public void SerializeAnalogCTagMetadata_MatchesCapturedNativeBytesByteForByte() { - // Reproduces the captured native EnsT2(Float) CTagMetadata bytes from - // artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/ - // fresh-enst2-latest.ndjson — 144 bytes. Inputs: - // tagName = "RetestSdkWriteSandbox" (the sandbox) - // description = "SDK write-RE sandbox tag" - // eu = "test" - // FILETIME = 0x01DCDBBFCD87D049 (captured at run time) - // The earlier 146-byte version mistakenly included the WCF EndElement closing - // markers (`01 01 01`) and was missing the 0x4E leading marker — both have been - // corrected by walking the native InBuff field-by-field. + // Reproduces the captured native EnsT2(Float) CTagMetadata bytes for the sandbox + // tag with default ranges and ApplyScaling=false. 2-byte trailer = `FE 00` where + // the second byte is the ApplyScaling flag (0x00 = false; 0x01 = true). const string ExpectedHex = "4E6703000100000004C6020100000000000000000000000000000000" + "09150052657465737453646B577269746553616E64626F78" @@ -118,6 +111,82 @@ public sealed class HistorianTagWriteProtocolTests Assert.Equal(Convert.ToHexString(expected), Convert.ToHexString(actual)); } + [Fact] + public void SerializeAnalogCTagMetadata_NonDefaultStorageRate_EncodesUInt32LittleEndianAtKnownOffset() + { + byte[] defaultRate = HistorianTagWriteProtocol.SerializeAnalogCTagMetadata( + tagName: "RetestSdkWriteRate", + description: "SDK write-RE sandbox tag", + engineeringUnit: "test", + dateCreatedUtc: DateTime.FromFileTimeUtc(0x01dcdbed474b29beL)); + byte[] customRate = HistorianTagWriteProtocol.SerializeAnalogCTagMetadata( + tagName: "RetestSdkWriteRate", + description: "SDK write-RE sandbox tag", + engineeringUnit: "test", + dateCreatedUtc: DateTime.FromFileTimeUtc(0x01dcdbed474b29beL), + storageRateMs: 2500u); + + Assert.Equal(defaultRate.Length, customRate.Length); + // Storage-rate uint32 is at the byte position immediately after the + // "MDAS" + flag-block sequence; the only diff between the two payloads + // is those 4 bytes. + int firstDiff = 0; + while (firstDiff < defaultRate.Length && defaultRate[firstDiff] == customRate[firstDiff]) firstDiff++; + Assert.Equal(0xE8, defaultRate[firstDiff]); // 1000 = 0x000003E8 LE → 0xE8 0x03 0x00 0x00 + Assert.Equal(0x03, defaultRate[firstDiff + 1]); + Assert.Equal(0xC4, customRate[firstDiff]); // 2500 = 0x000009C4 LE → 0xC4 0x09 0x00 0x00 + Assert.Equal(0x09, customRate[firstDiff + 1]); + // Beyond the 4-byte rate field, the rest is identical. + Assert.Equal( + Convert.ToHexString(defaultRate.AsSpan(firstDiff + 4)), + Convert.ToHexString(customRate.AsSpan(firstDiff + 4))); + } + + [Fact] + public void SerializeAnalogCTagMetadata_ZeroStorageRate_Throws() + { + Assert.Throws(() => HistorianTagWriteProtocol.SerializeAnalogCTagMetadata( + tagName: "RetestSdkWriteRate", + description: "x", + engineeringUnit: "test", + dateCreatedUtc: DateTime.UtcNow, + storageRateMs: 0u)); + } + + [Fact] + public void SerializeAnalogCTagMetadata_ApplyScalingTrue_FlipsTrailerSecondByte() + { + // Captured 2026-05-04 by toggling --write-apply-scaling on the native harness: + // ApplyScaling=true sets the trailer's second byte to 0x01 (vs 0x00 for false). + // Live-verified: with 0x01 the server persists distinct MinRaw/MaxRaw and sets + // AnalogTag.Scaling=1; with 0x00 it mirrors MinRaw to MinEU and sets Scaling=0. + byte[] withFlag = HistorianTagWriteProtocol.SerializeAnalogCTagMetadata( + tagName: "RetestSdkWriteFloatRanges", + description: "SDK write-RE sandbox tag", + engineeringUnit: "test", + dateCreatedUtc: DateTime.FromFileTimeUtc(0x01dcdbed474b29beL), + dataType: AVEVA.Historian.Client.Models.HistorianDataType.Float, + minEU: -50.0, maxEU: 200.0, minRaw: 10.0, maxRaw: 4095.0, + applyScaling: true); + byte[] withoutFlag = HistorianTagWriteProtocol.SerializeAnalogCTagMetadata( + tagName: "RetestSdkWriteFloatRanges", + description: "SDK write-RE sandbox tag", + engineeringUnit: "test", + dateCreatedUtc: DateTime.FromFileTimeUtc(0x01dcdbed474b29beL), + dataType: AVEVA.Historian.Client.Models.HistorianDataType.Float, + minEU: -50.0, maxEU: 200.0, minRaw: 10.0, maxRaw: 4095.0, + applyScaling: false); + + Assert.Equal(withoutFlag.Length, withFlag.Length); + Assert.Equal(0xFE, withFlag[^2]); + Assert.Equal(0x01, withFlag[^1]); + Assert.Equal(0xFE, withoutFlag[^2]); + Assert.Equal(0x00, withoutFlag[^1]); + Assert.Equal( + Convert.ToHexString(withoutFlag.AsSpan(0, withoutFlag.Length - 1)), + Convert.ToHexString(withFlag.AsSpan(0, withFlag.Length - 1))); + } + [Fact] public void GetAnalogDataTypeCode_UnsupportedType_Throws() { diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index a3405c5..ad274b4 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -237,6 +237,7 @@ internal static class Program // server-cache refresh after a fresh AddTag requires a NEW process / connection. bool skipAddTag = HasFlag(args, "--write-skip-add-tag"); bool skipAddValue = HasFlag(args, "--write-skip-add-value"); + bool writeApplyScaling = HasFlag(args, "--write-apply-scaling"); // Decoded via dnlib — actual enum field types on HistorianTag: // set_TagDataType stfld ArchestrA.HistorianDataType HistorianTag::dataType @@ -260,7 +261,7 @@ internal static class Program SetProperty(tag, "MinRaw", writeMinRaw); SetProperty(tag, "MaxRaw", writeMaxRaw); SetProperty(tag, "StorageRate", 1000u); - SetProperty(tag, "ApplyScaling", false); + SetProperty(tag, "ApplyScaling", writeApplyScaling); uint tagKey = 0; if (!skipAddTag) @@ -305,6 +306,25 @@ internal static class Program tagKey = realKey; } } + + using System.Data.SqlClient.SqlCommand analogCmd = sql.CreateCommand(); + analogCmd.CommandText = "SELECT MinEU, MaxEU, MinRaw, MaxRaw, Scaling FROM AnalogTag WHERE TagName = @t"; + analogCmd.Parameters.AddWithValue("@t", sandboxTag); + using System.Data.SqlClient.SqlDataReader analogReader = analogCmd.ExecuteReader(); + if (analogReader.Read()) + { + rows.Add(new + { + Kind = "AnalogTagPersisted", + TagName = sandboxTag, + MinEU = analogReader.IsDBNull(0) ? (object)"" : analogReader.GetDouble(0), + MaxEU = analogReader.IsDBNull(1) ? (object)"" : analogReader.GetDouble(1), + MinRaw = analogReader.IsDBNull(2) ? (object)"" : analogReader.GetDouble(2), + MaxRaw = analogReader.IsDBNull(3) ? (object)"" : analogReader.GetDouble(3), + Scaling = analogReader.IsDBNull(4) ? (object)"" : analogReader.GetValue(4), + InputApplyScaling = writeApplyScaling, + }); + } } // Server cache may not pick up new tags immediately. Allow a wait between AddTag