From 08b950caee93689ce2a942a5637d5186815911ef Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sun, 21 Jun 2026 01:43:19 -0400 Subject: [PATCH] R1.11 AddTagExtendedPropertiesAsync: extended-property write via AddTEx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds user-defined extended properties to an existing tag via the 2020 WCF AddTEx (AddTagExtendedProperties) op. Write-enabled connection + uppercase storage-session GUID handle; reuses the write orchestrator open/priming chain. The AddTEx inBuff is the exact inverse of the R1.5 GetTepByNm read-response framing, so the serializer mirrors the read parser: uint32 groupCount + 0x01(group) + [0x09+u16+ASCII tag] + uint32 propCount + per prop{ 0x02 + [0x09+u16+ASCII name] + 0x43 VT_BSTR + u16 payloadLen + u16 charCount + UTF-16 value } + 0x01(group trailer) + 0x00(terminator). The trailing 0x00 is required — without it inBuff is one byte short and the server throws SErrorException in CHistStorage::AddTagExtendedProperties. The golden fixture pins the clean inBuff the live server accepted (dumped via AVEVA_HISTORIAN_TEP_DUMP); read-back verified via R1.5. String (0x43) values only. Delete (DelTep) is deferred: the native DeleteTagExtendedPropertiesByName does a client-side sync check and returns err 229 for a just-added property, so the DelTep request never reaches the wire and its inBuff can't be captured yet. Shipped: HistorianClient.AddTagExtendedPropertiesAsync/AddTagExtendedPropertyAsync; HistorianTagExtendedPropertyProtocol.SerializeAddRequest; orchestrator path; golden WcfTagExtendedPropertyWriteProtocolTests (4); gated live write/read-back test; native-harness `add-tep` scenario + Capture-AddTagExtendedProperties.ps1 + decode-add-tep-capture.py. Doc: wcf-add-tag-extended-properties.md. 233 tests green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- docs/plans/hcal-capability-matrix.md | 2 +- docs/plans/hcal-roadmap.md | 2 +- .../wcf-add-tag-extended-properties.md | 72 ++++++++++ scripts/Capture-AddTagExtendedProperties.ps1 | 104 ++++++++++++++ scripts/decode-add-tep-capture.py | 115 ++++++++++++++++ src/AVEVA.Historian.Client/HistorianClient.cs | 22 +++ .../HistorianTagExtendedPropertyProtocol.cs | 77 +++++++++++ .../Wcf/HistorianWcfTagWriteOrchestrator.cs | 49 +++++++ .../HistorianClientIntegrationTests.cs | 60 ++++++++ ...cfTagExtendedPropertyWriteProtocolTests.cs | 72 ++++++++++ .../Program.cs | 129 +++++++++++++++++- 11 files changed, 701 insertions(+), 3 deletions(-) create mode 100644 docs/reverse-engineering/wcf-add-tag-extended-properties.md create mode 100644 scripts/Capture-AddTagExtendedProperties.ps1 create mode 100644 scripts/decode-add-tep-capture.py create mode 100644 tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs diff --git a/docs/plans/hcal-capability-matrix.md b/docs/plans/hcal-capability-matrix.md index 283ec1f..9dc7e71 100644 --- a/docs/plans/hcal-capability-matrix.md +++ b/docs/plans/hcal-capability-matrix.md @@ -74,7 +74,7 @@ blob needs RE). | Create string/discrete tag | `AddTag` | `History.EnsureTags` | ⬜ | GATED/BOUNDED | native AddTag rejects these types server-side; needs different metadata path | | Delete tag(s) | `DeleteTags` | `History.DeleteTags` | ✅ | DONE | | | Rename tag(s) | `RenameTags` | (History op) | ⬜ | BOUNDED | `AllowRenameTags` param already probed | -| Add/Delete extended properties | `AddTagExtendedProperties`, `DeleteTagExtendedPropertiesByName` | `History.AddTagExtendedProperties` / `DeleteTagExtendedProperties` | ⬜ | BOUNDED | gRPC op + TEP serialize | +| Add/Delete extended properties | `AddTagExtendedProperties`, `DeleteTagExtendedPropertiesByName` | `History.AddTagExtendedProperties` / `DeleteTagExtendedProperties` | 🟗 | BOUNDED | **Add DONE** (`AddTagExtendedPropertiesAsync`, AddTEx; inBuff = inverse of R1.5 read framing + trailing `01 00`). Delete (DelTep) deferred — native sync gate (err 229) blocks capturing its inBuff. See `wcf-add-tag-extended-properties.md` | | Add/Delete localized properties | `AddTagLocalizedProperties`, `DeleteTagLocalizedPropertiesByName` | `History.AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` | ⬜ | BOUNDED | | ## 6. Data writes — values diff --git a/docs/plans/hcal-roadmap.md b/docs/plans/hcal-roadmap.md index e459cbd..f845e89 100644 --- a/docs/plans/hcal-roadmap.md +++ b/docs/plans/hcal-roadmap.md @@ -121,7 +121,7 @@ read/browse/status surface is Windows-free and the gRPC stack is the default pat | ID | Capability | gRPC op | Payload | Notes | |---|---|---|---|---| | R1.10 | `RenameTagsAsync` | History rename op | rename request buffer | `AllowRenameTags` already probed | -| R1.11 | Extended-property **write** | `History.AddTagExtendedProperties` (+ groups) / `DeleteTagExtendedProperties` | TEP serialize | mirror analog CTagMetadata discipline | +| ~~R1.11~~ | Extended-property **write** | `History.AddTagExtendedProperties` (AddTEx) | ✅ **Add DONE (2026-06-21), live-verified.** `AddTagExtendedPropertiesAsync`/`AddTagExtendedPropertyAsync` (write mode, uppercase handle). inBuff = exact inverse of the R1.5 read framing (`uint32 groupCount + 0x01 + compact-ASCII tag + uint32 propCount + per prop[0x02 + compact-ASCII name + 0x43 VT_BSTR value] + 0x01 trailer + 0x00 terminator`); the trailing `0x00` is required or the server throws. Golden `WcfTagExtendedPropertyWriteProtocolTests` + gated live write/read-back test. **Delete (DelTep) deferred** — native client-side sync gate (err 229) blocks capturing its inBuff. See `docs/reverse-engineering/wcf-add-tag-extended-properties.md`. | | R1.12 | Localized-property **write** | `History.AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` | localized serialize | | | R1.13 | Non-analog tag create (string/discrete) | `History.EnsureTags` | distinct CTagMetadata variant | ⚠ native AddTag rejected some types — confirm server path first; may be GATED | diff --git a/docs/reverse-engineering/wcf-add-tag-extended-properties.md b/docs/reverse-engineering/wcf-add-tag-extended-properties.md new file mode 100644 index 0000000..2c5c076 --- /dev/null +++ b/docs/reverse-engineering/wcf-add-tag-extended-properties.md @@ -0,0 +1,72 @@ +# Extended-property write over 2020 WCF — AddTEx (HCAL R1.11) + +**Status: ✅ Add DONE + live-verified (2026-06-21). Delete (DelTep) deferred — see below.** +`HistorianClient.AddTagExtendedPropertiesAsync` / `AddTagExtendedPropertyAsync` writes user-defined +extended properties onto an existing tag via the 2020 WCF **`AddTEx`** (AddTagExtendedProperties) op, +and they read back via the R1.5 `GetTagExtendedPropertiesAsync` path. Verified end-to-end from the +pure-managed .NET 10 client against the local 2020 Historian (create tag → add property → read back → +delete tag). + +## The op + +``` +bool AddTagExtendedProperties(string handle, byte[] inBuff, out byte[] errorBuffer) // AddTEx +``` + +On `IHistoryServiceContract2` (History service). Requires a **write-enabled** connection (Open2 mode +`0x401`) and the uppercase storage-session GUID handle — the SDK reuses the write orchestrator's +open + priming chain (the same one used by EnsT2/DelT). The tag is referenced by name inside `inBuff`; +no extra per-connection tag registration was needed (the server resolves it). + +## The inBuff — the exact inverse of the R1.5 read response + +The native `AddTagExtendedProperties(TagExtendedPropertyGroupList, out err)` packs its groups into the +`AddTEx` `inBuff` with the **same framing the R1.5 `GetTepByNm` response uses**, so the write serializer +is the inverse of `HistorianTagExtendedPropertyProtocol.ParseResponse`: + +``` +uint32 groupCount (= 1) +byte 0x01 (group marker) +0x09 + uint16 byteLen + ASCII tagName (compact-ASCII string) +uint32 propertyCount +repeated propertyCount times: + byte 0x02 (property marker) + 0x09 + uint16 byteLen + ASCII propertyName + 0x43 + uint16 payloadLen + uint16 charCount + UTF-16LE value (VT_BSTR variant; payloadLen = 2 + charCount*2) +byte 0x01 (group trailer) +byte 0x00 (buffer terminator) +``` + +⚠️ **The trailing `0x01 0x00`** matters: the group trailer is `0x01` (as in the read parser) **plus a +final `0x00` buffer terminator**. Omitting the `0x00` makes `inBuff` one byte short and the server throws +`SErrorException` in `aahClientAccessPoint::CHistStorage::AddTagExtendedProperties` (AddTEx returns +false). The read parser tolerates the extra byte because it only consumes one trailing byte per group. + +Only the string (`0x43` VT_BSTR) value variant is evidence-backed (matching the read path). The raw +instrument capture mangles the final byte with MDAS chunk markers, so the golden fixture pins the +**clean** byte[] the SDK handed the channel (dumped via `AVEVA_HISTORIAN_TEP_DUMP`) — the exact buffer +the live server accepted. + +## Delete (DelTep) — deferred + +`DeleteTagExtendedProperties` (`DelTep`) is **not shipped yet**. The native +`DeleteTagExtendedPropertiesByName(tag, propertyNames, deleteFromServer, out err)` performs a +**client-side sync check** and returns error **229 ("Tag extended property not synchronized with +server")** when deleting a just-added property — so the `DelTep` request never reached the wire and its +inBuff could not be captured. Capturing it needs a property that is already server-synchronized (add it +in one session, then delete in a later one). Left for a follow-up rather than shipping a guessed buffer. + +## Shipped surface + +- `HistorianClient.AddTagExtendedPropertiesAsync(tag, IReadOnlyList)` and + `AddTagExtendedPropertyAsync(tag, name, value)`. +- `HistorianTagExtendedPropertyProtocol.SerializeAddRequest` (the inBuff serializer; lives beside the + R1.5 read parser); orchestrator path in `HistorianWcfTagWriteOrchestrator`. +- Golden `WcfTagExtendedPropertyWriteProtocolTests` (pins the server-accepted buffer + layout); gated + live test `AddTagExtendedPropertiesAsync_AgainstLocalHistorian_WritesAndReadsBack`. + +## Capture / decode tooling + +`scripts/Capture-AddTagExtendedProperties.ps1` (native-harness `add-tep` scenario + +instrument-wcf-{write,read}message; sandbox-guarded create→add→[optional delete]) and +`scripts/decode-add-tep-capture.py`. diff --git a/scripts/Capture-AddTagExtendedProperties.ps1 b/scripts/Capture-AddTagExtendedProperties.ps1 new file mode 100644 index 0000000..511a4b9 --- /dev/null +++ b/scripts/Capture-AddTagExtendedProperties.ps1 @@ -0,0 +1,104 @@ +<# +.SYNOPSIS + Captures the native AVEVA client's AddTagExtendedProperties / DeleteTagExtendedProperties wire + traffic (HCAL roadmap R1.11) so the AddTEx / DelTep inBuff layout can be decoded, not guessed. + +.DESCRIPTION + Drives the .NET-Framework NativeTraceHarness's `add-tep` scenario against the live Historian with + an IL-rewritten copy of aahClientManaged.dll whose ClientMessageEncoder.WriteMessage AND + ReadMessage are instrumented. The harness opens a WRITE-enabled connection, creates a sandbox tag + (RetestSdkWrite...), and calls AddTagExtendedProperties(TagExtendedPropertyGroupList, out err) with + one string property (and DeleteTagExtendedPropertiesByName when -Delete). + + Decode with scripts/decode-add-tep-capture.py: the WCF.WriteMessage.Body whose op is AddTEx carries + the inBuff (tag name + property name/value); DelTep carries the delete inBuff (tag + property names). + + SAFETY: sandbox-guarded — the tag MUST start with 'RetestSdkWrite'. Default run leaves the tag + + property in place (unless -Delete); pass -Delete to also capture DelTep and remove the property. + +.NOTES + Artifacts are diagnostic and gitignored. Sanitize before copying into docs/. +#> +[CmdletBinding()] +param( + [string]$ServerName = "localhost", + [int]$TcpPort = 32568, + [string]$TepTag = "RetestSdkWriteTepTag", + [string]$PropName = "SdkTestProp", + [string]$PropValue = "SdkTestValue", + [switch]$Delete, + [string]$Configuration = "Debug" +) + +$ErrorActionPreference = "Stop" +$repoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $repoRoot +if (-not $TepTag.StartsWith("RetestSdkWrite")) { throw "-TepTag must start with 'RetestSdkWrite' (sandbox guard)." } + +$reProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseEngineering\AVEVA.Historian.ReverseEngineering.csproj" +$harnessProj = Join-Path $repoRoot "tools\AVEVA.Historian.NativeTraceHarness\AVEVA.Historian.NativeTraceHarness.csproj" +$instrProj = Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\AVEVA.Historian.ReverseInstrumentation.csproj" + +$captureDir = Join-Path $repoRoot "artifacts\reverse-engineering\instrumented-wcf-add-tep" +$currentCopy = Join-Path $captureDir "current-copy" +$instrDll = Join-Path $captureDir "aahClientManaged.dll" +$capturePath = Join-Path $captureDir "add-tep-capture-latest.ndjson" + +Write-Host "== Building tooling ($Configuration) ==" -ForegroundColor Cyan +dotnet build $reProj -c $Configuration --nologo -v q | Out-Null +dotnet build $instrProj -c $Configuration --nologo -v q | Out-Null +dotnet build $harnessProj -c $Configuration --nologo -v q | Out-Null + +$instrSourceDll = Get-ChildItem -Recurse (Join-Path $repoRoot "tools\AVEVA.Historian.ReverseInstrumentation\bin\$Configuration") ` + -Filter "AVEVA.Historian.ReverseInstrumentation.dll" | Select-Object -First 1 -ExpandProperty FullName +if (-not $instrSourceDll) { throw "ReverseInstrumentation.dll not found under bin\$Configuration." } + +Write-Host "== Instrumenting WriteMessage + ReadMessage ==" -ForegroundColor Cyan +New-Item -ItemType Directory -Force -Path $captureDir | Out-Null +$writeOnly = Join-Path $captureDir "aahClientManaged.write.dll" +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-writemessage (Join-Path $repoRoot "current\aahClientManaged.dll") $writeOnly | Out-Null +dotnet run --no-build -c $Configuration --project $reProj -- ` + instrument-wcf-readmessage $writeOnly $instrDll | Out-Null + +Write-Host "== Staging current-copy ==" -ForegroundColor Cyan +robocopy (Join-Path $repoRoot "current") $currentCopy /MIR /NJH /NJS /NDL /NP /NC /NS | Out-Null +Copy-Item -Force $instrDll (Join-Path $currentCopy "aahClientManaged.dll") +Copy-Item -Force $instrSourceDll (Join-Path $currentCopy "AVEVA.Historian.ReverseInstrumentation.dll") + +$harnessDll = Join-Path $currentCopy "aahClientManaged.dll" +if (Test-Path $capturePath) { Remove-Item -Force $capturePath } +$env:AVEVA_HISTORIAN_RE_CAPTURE = $capturePath + +Write-Host "== Capturing add-tep ($TepTag : $PropName=$PropValue) ==" -ForegroundColor Green +$harnessArgs = @( + "--scenario", "add-tep", + "--server-name", $ServerName, + "--tcp-port", "$TcpPort", + "--tep-tag", $TepTag, + "--tep-name", $PropName, + "--tep-value", $PropValue, + "--current-dir", $currentCopy, + "--managed-dll-path", $harnessDll +) +if ($Delete) { $harnessArgs += "--tep-delete" } + +$harnessJson = $null +try { + $prevEap = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $harnessJson = & dotnet run --no-build -c $Configuration --project $harnessProj -- @harnessArgs 2>&1 +} catch { + Write-Host " (add-tep raised: $($_.Exception.Message))" -ForegroundColor Yellow +} finally { + $ErrorActionPreference = $prevEap +} + +Remove-Item Env:\AVEVA_HISTORIAN_RE_CAPTURE -ErrorAction SilentlyContinue + +$recCount = if (Test-Path $capturePath) { (Get-Content $capturePath | Where-Object { $_.Trim() }).Count } else { 0 } +Write-Host "`n== Capture summary ==" -ForegroundColor Cyan +Write-Host " -> $recCount records -> $capturePath" +Write-Host "Harness output (AddTagExtendedProperties / Rows):" -ForegroundColor Cyan +$harnessJson | Select-Object -Last 24 +Write-Host "`nDecode with: python scripts\decode-add-tep-capture.py" -ForegroundColor Cyan diff --git a/scripts/decode-add-tep-capture.py b/scripts/decode-add-tep-capture.py new file mode 100644 index 0000000..f3692bc --- /dev/null +++ b/scripts/decode-add-tep-capture.py @@ -0,0 +1,115 @@ +"""Decode the AddTagExtendedProperties / DeleteTagExtendedProperties WCF inBuff (HCAL R1.11). + +Reads the capture produced by scripts/Capture-AddTagExtendedProperties.ps1 and locates the AddTEx / +DelTep WriteMessage bodies by the sandbox tag + property name/value, then dumps the inBuff bytes so +the framing (tag name, property count, per-property name + value markers) can be read off. Compare to +the R1.5 read-response encoding in HistorianTagExtendedPropertyProtocol. + +Output is diagnostic. Sanitize before copying into docs/. +""" +import base64 +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CAPDIR = REPO_ROOT / "artifacts" / "reverse-engineering" / "instrumented-wcf-add-tep" +CAP = CAPDIR / "add-tep-capture-latest.ndjson" + +TAG = "RetestSdkWriteTepTag" +PROP = "SdkTestProp" +VALUE = "SdkTestValue" +OP_ADD = b"AddTEx" +OP_DEL = b"DelTep" + + +def hexdump(label, buf, base=0): + print(f"=== {label}: {len(buf)} bytes ===") + for off in range(0, len(buf), 16): + c = buf[off:off + 16] + hp = " ".join(f"{x:02X}" for x in c) + ap = "".join(chr(x) if 32 <= x < 127 else "." for x in c) + print(f" {base + off:04X} {hp:<48} |{ap}|") + print() + + +def ascii_strings(buf, minlen=3): + out, cur, start = [], [], 0 + for i, x in enumerate(buf): + if 32 <= x < 127: + if not cur: + start = i + cur.append(chr(x)) + else: + if len(cur) >= minlen: + out.append((start, "".join(cur))) + cur = [] + if len(cur) >= minlen: + out.append((start, "".join(cur))) + return out + + +def u16_strings(buf, minlen=3): + out, i = [], 0 + while i < len(buf) - 1: + j, chars = i, [] + while j < len(buf) - 1 and 32 <= buf[j] < 127 and buf[j + 1] == 0: + chars.append(chr(buf[j])) + j += 2 + if len(chars) >= minlen: + out.append((i, "".join(chars))) + i = j + else: + i += 1 + return out + + +def main() -> int: + if not CAP.exists(): + print(f"Missing capture: {CAP}\nRun scripts/Capture-AddTagExtendedProperties.ps1 first.") + return 1 + + records = [] + for line in CAP.open(encoding="utf-8-sig"): + if line.strip(): + records.append(json.loads(line)) + + tag_a, prop_a, val_a = TAG.encode("ascii"), PROP.encode("ascii"), VALUE.encode("ascii") + tag_u, prop_u, val_u = TAG.encode("utf-16-le"), PROP.encode("utf-16-le"), VALUE.encode("utf-16-le") + + print(f"== {len(records)} MDAS bodies captured ==") + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + flags = [] + if OP_ADD in body: + flags.append("AddTEx") + if OP_DEL in body: + flags.append("DelTep") + if prop_a in body or prop_u in body: + flags.append("PROP") + if val_a in body or val_u in body: + flags.append("VALUE") + print(f" [{idx:02d}] {rec.get('Phase'):26s} len={len(body):5d} {','.join(flags)}") + + def dump(op): + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + if rec.get("Phase") == "WCF.WriteMessage.Body" and op in body: + hexdump(f"[{idx}] {op.decode()} WriteMessage", body) + print(" UTF-16 strings:") + for off, s in u16_strings(body): + print(f" 0x{off:04X} {s!r}") + print(" ASCII strings:") + for off, s in ascii_strings(body): + print(f" 0x{off:04X} {s!r}") + print() + + print("\n== AddTEx request(s) ==") + dump(OP_ADD) + print("\n== DelTep request(s) ==") + dump(OP_DEL) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/AVEVA.Historian.Client/HistorianClient.cs b/src/AVEVA.Historian.Client/HistorianClient.cs index 54b00f6..fa72656 100644 --- a/src/AVEVA.Historian.Client/HistorianClient.cs +++ b/src/AVEVA.Historian.Client/HistorianClient.cs @@ -179,6 +179,28 @@ public sealed class HistorianClient : IAsyncDisposable return _protocol.GetTagExtendedPropertiesAsync(tag, cancellationToken); } + /// + /// Adds (or updates) extended (user-defined) properties on an existing tag via the 2020 WCF + /// AddTEx (AddTagExtendedProperties) op. Requires a write-enabled connection. String-valued + /// properties only (the evidence-backed surface). The new properties are read back via + /// . See HistorianTagExtendedPropertyProtocol. + /// + public Task AddTagExtendedPropertiesAsync(string tag, IReadOnlyList properties, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tag); + ArgumentNullException.ThrowIfNull(properties); + return new HistorianWcfTagWriteOrchestrator(_options).AddTagExtendedPropertiesAsync(tag, properties, cancellationToken); + } + + /// Convenience overload of for a single + /// string-valued property. + public Task AddTagExtendedPropertyAsync(string tag, string name, string value, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tag); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + return AddTagExtendedPropertiesAsync(tag, [new HistorianTagExtendedProperty(name, value ?? string.Empty)], cancellationToken); + } + /// /// Creates or updates the named tag in the Historian Runtime database via /// EnsureTags2. Currently only is diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs index ef42330..dd5aa69 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianTagExtendedPropertyProtocol.cs @@ -55,6 +55,83 @@ internal static class HistorianTagExtendedPropertyProtocol return stream.ToArray(); } + private const byte GroupMarker = 0x01; + private const byte PropertyMarker = 0x02; + + /// + /// Serializes the AddTEx (AddTagExtendedProperties) inBuff for a single tag's string + /// properties. The buffer is the exact inverse of the group framing + /// (decoded from a native AddTagExtendedProperties capture; see + /// docs/reverse-engineering/wcf-add-tag-extended-properties.md): + /// + /// uint32 groupCount (= 1) + /// byte 0x01 (group marker) + /// 0x09 + uint16 byteLen + ASCII tagName + /// uint32 propertyCount + /// repeated: byte 0x02 (property marker) + 0x09 + uint16 byteLen + ASCII propName + /// + 0x43 VT_BSTR + uint16 payloadLen + uint16 charCount + UTF-16LE value + /// byte 0x01 (group trailer) + /// + /// Only string-valued properties are evidence-backed (the VT_BSTR variant), matching the read path. + /// + public static byte[] SerializeAddRequest(string tagName, IReadOnlyList properties) + { + ArgumentException.ThrowIfNullOrEmpty(tagName); + ArgumentNullException.ThrowIfNull(properties); + if (properties.Count == 0) + { + throw new ArgumentException("At least one extended property is required.", nameof(properties)); + } + + using MemoryStream stream = new(); + Span u32 = stackalloc byte[4]; + + BinaryPrimitives.WriteUInt32LittleEndian(u32, 1u); // group count + stream.Write(u32); + + stream.WriteByte(GroupMarker); + WriteCompactAscii(stream, tagName); + + BinaryPrimitives.WriteUInt32LittleEndian(u32, checked((uint)properties.Count)); + stream.Write(u32); + + foreach (HistorianTagExtendedProperty property in properties) + { + ArgumentException.ThrowIfNullOrEmpty(property?.Name, nameof(properties)); + stream.WriteByte(PropertyMarker); + WriteCompactAscii(stream, property.Name); + WriteVariantString(stream, property.Value ?? string.Empty); + } + + stream.WriteByte(GroupMarker); // group trailer + stream.WriteByte(0x00); // buffer terminator (captured: the native inBuff ends 0x01 0x00) + return stream.ToArray(); + } + + private static void WriteCompactAscii(MemoryStream stream, string value) + { + byte[] ascii = Encoding.ASCII.GetBytes(value); + stream.WriteByte(CompactStringMarker); + Span u16 = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(u16, checked((ushort)ascii.Length)); + stream.Write(u16); + stream.Write(ascii, 0, ascii.Length); + } + + private static void WriteVariantString(MemoryStream stream, string value) + { + byte[] utf16 = Encoding.Unicode.GetBytes(value); + ushort charCount = checked((ushort)value.Length); + stream.WriteByte(VariantTypeBStr); + Span u16 = stackalloc byte[2]; + // payloadLen = the bytes that follow it: the uint16 charCount field (2) + UTF-16 bytes. + BinaryPrimitives.WriteUInt16LittleEndian(u16, checked((ushort)(2 + utf16.Length))); + stream.Write(u16); + BinaryPrimitives.WriteUInt16LittleEndian(u16, charCount); + stream.Write(u16); + stream.Write(utf16, 0, utf16.Length); + } + /// /// Parses the tagExtendedProperties response buffer into a flat list of /// (tagName, propertyName, value) rows. Returns an empty list when the buffer carries no rows diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs index b43f72a..bd47048 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfTagWriteOrchestrator.cs @@ -41,6 +41,18 @@ internal sealed class HistorianWcfTagWriteOrchestrator return Task.Run(() => DeleteTag(tagName), cancellationToken); } + public Task AddTagExtendedPropertiesAsync( + string tagName, IReadOnlyList properties, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tagName); + ArgumentNullException.ThrowIfNull(properties); + if (properties.Count == 0) + { + throw new ArgumentException("At least one extended property is required.", nameof(properties)); + } + return Task.Run(() => AddTagExtendedProperties(tagName, properties), cancellationToken); + } + private bool EnsureTag(HistorianTagDefinition definition) { Guid contextKey = Guid.NewGuid(); @@ -86,6 +98,43 @@ internal sealed class HistorianWcfTagWriteOrchestrator return result; } + private bool AddTagExtendedProperties(string tagName, IReadOnlyList properties) + { + Guid contextKey = Guid.NewGuid(); + var (histBinding, histEndpoint, _, _) = HistorianWcfBindingFactory.CreateBindingPair(_options); + Binding auxBinding = HistorianWcfBindingFactory.CreateAuxiliaryBinding(_options); + EndpointAddress statusEndpoint = HistorianWcfBindingFactory.CreateAuxiliaryEndpointAddress(_options, HistorianWcfServiceNames.Status); + EndpointAddress transactionEndpoint = HistorianWcfBindingFactory.CreateAuxiliaryEndpointAddress(_options, HistorianWcfServiceNames.Transaction); + EndpointAddress retrievalEndpoint = _options.Transport == HistorianTransport.LocalPipe + ? HistorianWcfBindingFactory.CreatePipeEndpointAddress(_options.Host, HistorianWcfServiceNames.Retrieval) + : HistorianWcfBindingFactory.CreateEndpointAddress(_options.Host, _options.Port, HistorianWcfServiceNames.Retrieval); + + bool result = false; + HistorianWcfAuthChainHelper.OpenAuthenticatedConnection( + _options, histBinding, histEndpoint, contextKey, CancellationToken.None, + connectionMode: HistorianWcfAuthChainHelper.NativeIntegratedWriteEnabledConnectionMode, + additionalSetup: (historyChannel, context) => + { + RunWritePriming(historyChannel, context, auxBinding, statusEndpoint, transactionEndpoint, retrievalEndpoint); + string handle = context.StorageSessionId.ToString("D").ToUpperInvariant(); + byte[] inBuff = HistorianTagExtendedPropertyProtocol.SerializeAddRequest(tagName, properties); + DumpTepIfRequested(inBuff); + bool ok = historyChannel.AddTagExtendedProperties(handle, inBuff, out byte[] errorBuffer); + WriteDiag("AddTEx", $"Returned={ok} Tag={tagName} PropCount={properties.Count} InLen={inBuff.Length} ErrLen={errorBuffer?.Length ?? -1} ErrHex={(errorBuffer is null ? "" : Convert.ToHexString(errorBuffer))}"); + result = ok; + }); + return result; + } + + /// Env-gated dump of the clean AddTEx inBuff (base64) for golden-fixture capture, + /// mirroring the rename/SQL dump hooks — avoids hand-stitching MDAS chunk markers. + private static void DumpTepIfRequested(byte[] inBuff) + { + string? path = Environment.GetEnvironmentVariable("AVEVA_HISTORIAN_TEP_DUMP"); + if (string.IsNullOrWhiteSpace(path)) return; + try { File.AppendAllText(path, Convert.ToBase64String(inBuff) + Environment.NewLine); } catch { } + } + private static bool SendEnsureTags2( IHistoryServiceContract2 historyChannel, HistorianWcfAuthChainHelper.OpenConnectionContext context, diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs index 18323e6..82ab6e2 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs @@ -505,6 +505,66 @@ public sealed class HistorianClientIntegrationTests Assert.True(deleted, "DeleteTagAsync returned false against the live Historian."); } + [Fact] + public async Task AddTagExtendedPropertiesAsync_AgainstLocalHistorian_WritesAndReadsBack() + { + // Safety: localhost only, sandbox tag must start with "RetestSdkWrite". Creates the tag, + // adds an extended property, reads it back via R1.5, and deletes the tag. Gated on + // HISTORIAN_WRITE_SANDBOX_TAG. + string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); + string? sandboxTag = Environment.GetEnvironmentVariable("HISTORIAN_WRITE_SANDBOX_TAG"); + if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) + { + return; + } + if (string.IsNullOrWhiteSpace(sandboxTag) || !sandboxTag.StartsWith("RetestSdkWrite", StringComparison.Ordinal)) + { + return; // safety gate + } + + HistorianClient client = new(new HistorianClientOptions + { + Host = host, + IntegratedSecurity = true, + Transport = HistorianTransport.LocalPipe + }); + + await client.EnsureTagAsync(new AVEVA.Historian.Client.Models.HistorianTagDefinition + { + TagName = sandboxTag, + Description = "SDK ext-property write live test", + EngineeringUnit = "test", + DataType = AVEVA.Historian.Client.Models.HistorianDataType.Float, + MinEU = 0.0, + MaxEU = 100.0, + }, CancellationToken.None); + + try + { + const string propName = "SdkLiveTestProp"; + const string propValue = "SdkLiveTestValue"; + + bool added = await client.AddTagExtendedPropertyAsync(sandboxTag, propName, propValue, CancellationToken.None); + Assert.True(added, "AddTagExtendedPropertyAsync returned false against the live Historian."); + + // Read back via R1.5 (server may take a moment to surface the new property). + bool found = false; + for (int i = 0; i < 10 && !found; i++) + { + await Task.Delay(500); + var props = await client.GetTagExtendedPropertiesAsync(sandboxTag, CancellationToken.None); + found = props.Any(p => + string.Equals(p.Name, propName, StringComparison.OrdinalIgnoreCase) && + string.Equals(p.Value, propValue, StringComparison.Ordinal)); + } + Assert.True(found, $"Extended property '{propName}={propValue}' was not read back after the write."); + } + finally + { + try { await client.DeleteTagAsync(sandboxTag, CancellationToken.None); } catch { } + } + } + // Round-trip every live-verified analog data type + the non-default-range case. The // sandbox tag name is suffixed per case so the runs don't collide. Always cleans up. [Theory] diff --git a/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs new file mode 100644 index 0000000..0597eb0 --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/WcfTagExtendedPropertyWriteProtocolTests.cs @@ -0,0 +1,72 @@ +using System.Buffers.Binary; +using System.Text; +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Wcf; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Golden-byte tests for the AddTEx (AddTagExtendedProperties) inBuff serializer (HCAL R1.11). +/// The reference buffer is the exact byte[] the SDK handed the WCF channel in the live +/// AddTagExtendedPropertiesAsync_AgainstLocalHistorian_WritesAndReadsBack run — the server +/// accepted it and the property was read back, so it is server-validated. +/// +public sealed class WcfTagExtendedPropertyWriteProtocolTests +{ + // Server-accepted AddTEx inBuff for tag "RetestSdkWriteTepSdk3", property "SdkLiveTestProp" = + // "SdkLiveTestValue", dumped via AVEVA_HISTORIAN_TEP_DUMP during the live write test. + private const string ServerAcceptedInBuffBase64 = + "AQAAAAEJFQBSZXRlc3RTZGtXcml0ZVRlcFNkazMBAAAAAgkPAFNka0xpdmVUZXN0UHJvcEMiABAAUwBkAGsATABpAHYAZQBUAGUAcwB0AFYAYQBsAHUAZQABAA=="; + + [Fact] + public void SerializeAddRequest_MatchesServerAcceptedBuffer() + { + byte[] expected = Convert.FromBase64String(ServerAcceptedInBuffBase64); + byte[] actual = HistorianTagExtendedPropertyProtocol.SerializeAddRequest( + "RetestSdkWriteTepSdk3", + [new HistorianTagExtendedProperty("SdkLiveTestProp", "SdkLiveTestValue")]); + Assert.Equal(expected, actual); + } + + [Fact] + public void SerializeAddRequest_SingleProperty_HasExpectedLayout() + { + byte[] buf = HistorianTagExtendedPropertyProtocol.SerializeAddRequest( + "ReactorTemp", [new HistorianTagExtendedProperty("Location", "PlantA")]); + + int c = 0; + Assert.Equal(1u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(c, 4))); c += 4; // group count + Assert.Equal(0x01, buf[c++]); // group marker + Assert.Equal(0x09, buf[c++]); // compact string marker + Assert.Equal(11, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; // tag byte len + Assert.Equal("ReactorTemp", Encoding.ASCII.GetString(buf.AsSpan(c, 11))); c += 11; + Assert.Equal(1u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(c, 4))); c += 4; // property count + Assert.Equal(0x02, buf[c++]); // property marker + Assert.Equal(0x09, buf[c++]); + Assert.Equal(8, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; + Assert.Equal("Location", Encoding.ASCII.GetString(buf.AsSpan(c, 8))); c += 8; + Assert.Equal(0x43, buf[c++]); // VT_BSTR + Assert.Equal(14, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; // payloadLen = 2 + 6*2 + Assert.Equal(6, BinaryPrimitives.ReadUInt16LittleEndian(buf.AsSpan(c, 2))); c += 2; // char count + Assert.Equal("PlantA", Encoding.Unicode.GetString(buf.AsSpan(c, 12))); c += 12; + Assert.Equal(0x01, buf[c++]); // group trailer + Assert.Equal(0x00, buf[c++]); // buffer terminator + Assert.Equal(buf.Length, c); + } + + [Fact] + public void SerializeAddRequest_MultipleProperties_EncodesCount() + { + byte[] buf = HistorianTagExtendedPropertyProtocol.SerializeAddRequest( + "T", [new HistorianTagExtendedProperty("A", "1"), new HistorianTagExtendedProperty("B", "2")]); + // group count @0, then 0x01 marker, 0x09 + u16(1) + "T" => property count at offset 4+1+1+2+1 = 9 + Assert.Equal(2u, BinaryPrimitives.ReadUInt32LittleEndian(buf.AsSpan(9, 4))); + } + + [Fact] + public void SerializeAddRequest_NoProperties_Throws() + { + Assert.Throws(() => + HistorianTagExtendedPropertyProtocol.SerializeAddRequest("T", Array.Empty())); + } +} diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index 5d14c59..c135f06 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -131,7 +131,7 @@ internal static class Program object connectionArgs = Activator.CreateInstance(connectionArgsType)!; SetProperty(connectionArgs, "ServerName", serverName); SetProperty(connectionArgs, "TcpPort", checked((ushort)tcpPort)); - SetProperty(connectionArgs, "ReadOnly", !(IsWriteScenario(scenario) || IsEventSendScenario(scenario))); + SetProperty(connectionArgs, "ReadOnly", !(IsWriteScenario(scenario) || IsEventSendScenario(scenario) || IsAddTagExtendedPropertiesScenario(scenario))); SetProperty(connectionArgs, "IntegratedSecurity", integratedSecurity); SetProperty(connectionArgs, "ConnectionType", Enum.Parse(connectionType, IsEventConnectionScenario(scenario) ? "Event" : "Process")); if (directConnection) @@ -323,6 +323,120 @@ internal static class Program })); return 0; } + else if (openSuccess && status.ConnectedToServer && IsAddTagExtendedPropertiesScenario(scenario)) + { + // R1.11 capture: drive AddTagExtendedProperties(TagExtendedPropertyGroupList, out err) + // — and optionally DeleteTagExtendedPropertiesByName — so instrument-wcf-writemessage can + // observe the AddTEx / DelTep inBuff (tag + property name/value framing). Sandbox-guarded. + string tepTag = GetArg(args, "--tep-tag") ?? "RetestSdkWriteTepTag"; + if (!tepTag.StartsWith("RetestSdkWrite", StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "add-tep scenario refuses tags that don't start with 'RetestSdkWrite'. Pass --tep-tag RetestSdkWrite..."); + } + string propName = GetArg(args, "--tep-name") ?? "SdkTestProp"; + string propValue = GetArg(args, "--tep-value") ?? "SdkTestValue"; + + var tepRows = new List(); + + // 1) Ensure the tag exists (AddTag) unless --tep-skip-create. + if (!HasFlag(args, "--tep-skip-create")) + { + Type tagDefType = GetType(assembly, "ArchestrA.HistorianTag"); + Type tagDataTypeEnum = GetType(assembly, "ArchestrA.HistorianDataType"); + Type tagStorageTypeEnum = GetType(assembly, "ArchestrA.HistorianStorageType"); + object tag = Activator.CreateInstance(tagDefType)!; + SetProperty(tag, "TagName", tepTag); + SetProperty(tag, "TagDescription", "SDK ext-property write RE sandbox tag"); + SetProperty(tag, "EngineeringUnit", "test"); + SetProperty(tag, "TagDataType", Enum.Parse(tagDataTypeEnum, "Float", ignoreCase: true)); + SetProperty(tag, "TagStorageType", Enum.Parse(tagStorageTypeEnum, "Cyclic", ignoreCase: true)); + SetProperty(tag, "MinEU", 0.0); + SetProperty(tag, "MaxEU", 100.0); + SetProperty(tag, "MinRaw", 0.0); + SetProperty(tag, "MaxRaw", 100.0); + SetProperty(tag, "StorageRate", 1000u); + SetProperty(tag, "ApplyScaling", false); + object addError = Activator.CreateInstance(errorType)!; + MethodInfo addTagMethod = accessType.GetMethod("AddTag", new[] { tagDefType, typeof(uint).MakeByRefType(), errorType.MakeByRefType() })!; + object?[] addTagArgs = [tag, 0u, addError]; + bool addOk = (bool)addTagMethod.Invoke(access, addTagArgs)!; + tepRows.Add(new { Kind = "AddTag", Success = addOk, ErrorDescription = GetPropertyText(addTagArgs[2]!, "ErrorDescription") }); + } + + // Prime the tag identity (same reason as the read scenario — server-side resolution). + MethodInfo? getTagInfoByName = accessType.GetMethods() + .FirstOrDefault(m => m.Name == "GetTagInfoByName" && m.GetParameters().Length == 4); + if (getTagInfoByName is not null) + { + object tibError = Activator.CreateInstance(errorType)!; + object?[] tibArgs = new object?[] { tepTag, true, null, tibError }; + try { getTagInfoByName.Invoke(access, tibArgs); } catch { } + } + + // 2) Build TagExtendedPropertyGroupList { TagExtendedPropertyGroup { TagName, [TagExtendedProperty] } } + Type listType = GetType(assembly, "ArchestrA.TagExtendedPropertyGroupList"); + Type groupType = GetType(assembly, "ArchestrA.TagExtendedPropertyGroup"); + Type propType = GetType(assembly, "ArchestrA.TagExtendedProperty"); + Type propDataTypeEnum = GetType(assembly, "ArchestrA.TagExtendedPropertyDataType"); + + object list = Activator.CreateInstance(listType)!; + object group = Activator.CreateInstance(groupType)!; + SetProperty(group, "TagName", tepTag); + object prop = Activator.CreateInstance(propType)!; + SetProperty(prop, "PropertyName", propName); + SetProperty(prop, "Type", Enum.Parse(propDataTypeEnum, "String", ignoreCase: true)); + SetProperty(prop, "Value", propValue); + groupType.GetMethod("Add", new[] { propType })!.Invoke(group, [prop]); + listType.GetMethod("Add", new[] { groupType })!.Invoke(list, [group]); + + MethodInfo addTepMethod = accessType.GetMethods() + .First(m => m.Name == "AddTagExtendedProperties" && m.GetParameters().Length == 2); + object addTepError = Activator.CreateInstance(errorType)!; + object?[] addTepArgs = [list, addTepError]; + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tep"); + bool addTepOk = false; + string? addTepEx = null; + try { addTepOk = (bool)addTepMethod.Invoke(access, addTepArgs)!; } + catch (TargetInvocationException ex) { addTepEx = FormatException(ex.InnerException ?? ex); } + tepRows.Add(new + { + Kind = "AddTagExtendedProperties", + Success = addTepOk, + Exception = addTepEx, + ErrorDescription = GetPropertyText(addTepArgs[1]!, "ErrorDescription"), + ErrorCode = GetPropertyText(addTepArgs[1]!, "ErrorCode"), + }); + + // 3) Optional delete (DelTep) to capture its inBuff too. + if (HasFlag(args, "--tep-delete")) + { + MethodInfo? delTepMethod = accessType.GetMethods() + .FirstOrDefault(m => m.Name == "DeleteTagExtendedPropertiesByName" && m.GetParameters().Length == 4); + if (delTepMethod is not null) + { + Type namesColType = delTepMethod.GetParameters()[1].ParameterType; // StringCollection + object names = Activator.CreateInstance(namesColType)!; + namesColType.GetMethod("Add", new[] { typeof(string) })!.Invoke(names, [propName]); + object delErr = Activator.CreateInstance(errorType)!; + object?[] delArgs = [tepTag, names, true, delErr]; + bool delOk = false; string? delEx = null; + try { delOk = (bool)delTepMethod.Invoke(access, delArgs)!; } + catch (TargetInvocationException ex) { delEx = FormatException(ex.InnerException ?? ex); } + tepRows.Add(new { Kind = "DeleteTagExtendedPropertiesByName", Success = delOk, Exception = delEx, ErrorDescription = GetPropertyText(delArgs[3]!, "ErrorDescription") }); + } + } + + Console.WriteLine(Serialize(new + { + Scenario = scenario, + TepTag = tepTag, + PropertyName = propName, + PropertyValue = propValue, + Rows = tepRows, + })); + return 0; + } else if (openSuccess && status.ConnectedToServer && IsEventSendScenario(scenario)) { // R2.1 capture: drive AddStreamedValue(HistorianEvent) and let instrument-wcf-* @@ -1638,6 +1752,19 @@ internal static class Program || scenario.Equals("tag-tep", StringComparison.OrdinalIgnoreCase); } + /// + /// Extended-property WRITE scenario (R1.11 capture): opens a write-enabled connection and calls + /// AddTagExtendedProperties(TagExtendedPropertyGroupList, out err) (and optionally + /// DeleteTagExtendedPropertiesByName) so instrument-wcf-writemessage can observe the + /// AddTEx/DelTep inBuff. Sandbox-guarded: the tag must start with RetestSdkWrite. + /// + private static bool IsAddTagExtendedPropertiesScenario(string scenario) + { + return scenario.Equals("add-tep", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("tep-write", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("extended-property-write", StringComparison.OrdinalIgnoreCase); + } + private static bool IsEventConnectionScenario(string scenario) { return IsEventScenario(scenario) || IsEventSendScenario(scenario);