From b3d22befd0b5ff5f3d466e5c68eb9e0dd17fb449 Mon Sep 17 00:00:00 2001 From: dohertj2 Date: Mon, 4 May 2026 08:06:28 -0400 Subject: [PATCH] write-commands plan: AddS2 prereq is architectural - not implementable as generic client write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-up attempts to satisfy the AddS2 server-cache prereq all failed at the same client-side gate before any AddS2 byte reached the wire: 1. TagKey synthetic→real override. First attempt used the placeholder TagKey=10000000 returned by HistorianAccess.AddTag. Native AddStreamedValue refused with error 168 "Tag not added to server". Harness now ALWAYS resolves the real wwTagKey from Runtime.dbo.Tag after AddTag (logged as TagKeyOverride: Synthetic→RealFromSql). Error code shifted to 129 "Tag not found in cache" — request now reaches the server but the server's in-memory tag cache doesn't know about the new tag. 2. Server-cache settle wait. Up to 8s sleep between AddTag and AddStreamedValue (--write-resync-wait-seconds N). Wait period contains 2× UpdC3 + 2× Trx/GetV keep-alives but no server-side cache update — error 129 persists. 3. Fresh process / fresh connection. Skipped AddTag entirely (--write-skip-add-tag) and ran AddStreamedValue alone against the already-existing sandbox tag. New native client instance, new client-side cache, new server session. SAME error 129 — no AddS2 bytes sent on wire. Capture confirms 44 records ending in Close2. Interpretation: the Historian engine's runtime tag cache only ingests tags from configured IOServers / Application Server data pipelines, not from HistorianAccess.AddTag-only client flows. AddTag populates Runtime.dbo.Tag (wwTagKey=240 was created) but doesn't register the tag with the live cache that AddStreamedValue checks. That registration happens server-side when an upstream data producer (an OPC driver, AnE event subsystem, Application Server attribute store) claims the tag. WriteValueAsync therefore CANNOT be implemented as a generic client API against this server architecture. The SDK's realistic writeable surface is now narrowed to EnsureTagAsync + DeleteTagAsync only. Harness changes: - --write-skip-add-tag skip the AddTag call (for fresh-cache test) - --write-skip-add-value skip the AddStreamedValue call (capture EnsT2 only) - --write-resync-wait-seconds N sleep N seconds between AddTag and AddStreamedValue (default 0) - TagKey lookup now ALWAYS hits SQL after AddTag, not just when the synthetic key is 0. Plan doc updated with full Phase 2 follow-on findings + revised remaining work (4-item checklist focused on EnsureTagAsync/ DeleteTagAsync, plus a stretch goal of probing AddRevisionValues* against an existing-tag). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../write-commands-reverse-engineering.md | 85 ++++++++++++++++--- .../Program.cs | 67 ++++++++++----- 2 files changed, 119 insertions(+), 33 deletions(-) diff --git a/docs/plans/write-commands-reverse-engineering.md b/docs/plans/write-commands-reverse-engineering.md index 2dab732..0460229 100644 --- a/docs/plans/write-commands-reverse-engineering.md +++ b/docs/plans/write-commands-reverse-engineering.md @@ -86,19 +86,80 @@ Visible fields (still being decoded against the **Decoder script** at `scripts/decode-write-capture.py` for the next session. -## Phase 2 remaining work +## Phase 2 follow-on findings (2026-05-04, second pass) -1. Decode the AddS2 prereq — find what RegisterTag / AddTagPair call - the server expects between EnsT2 and AddS2. Likely - `aahClientCommon.CHistStorage.AddTagidPairs` (token `0x0600202F`) - or `AddTagsWithServerTagId` (token `0x06002026`). -2. Capture AddS2 wire bytes once the prereq is satisfied. -3. Implement `HistorianAddTagsProtocol.SerializeAnalogCTagMetadata` / - discrete / string variants from the 146-byte capture above. -4. Implement `HistorianAddStreamValuesProtocol.Serialize` from the - yet-to-capture AddS2 bytes. -5. Implement the public surface: `EnsureTagAsync`, `WriteValueAsync`, - `DeleteTagAsync` (golden-byte + gated live integration tests). +**The AddS2 prereq is architectural, not protocol-level.** Three follow-up +attempts to trigger AddS2 from the sandbox harness all hit a client-side +gate before any AddS2 byte reaches the wire: + +1. **TagKey synthetic→real override.** First attempt used the placeholder + `TagKey=10000000` returned by `HistorianAccess.AddTag`. Native + `AddStreamedValue` refused with error 168 "Tag not added to server". + The harness now ALWAYS resolves the real `wwTagKey` from + `Runtime.dbo.Tag` after AddTag (logged as `TagKeyOverride: Synthetic→RealFromSql`). + Result: error code shifts to **129 "Tag not found in cache"**. + +2. **Server-cache settle wait.** Inserted up to 8s sleep between AddTag and + AddStreamedValue (configurable via `--write-resync-wait-seconds`). The + wait period contains 2× UpdC3 + 2× Trx/GetV keep-alives but no + server-side cache update — error 129 persists. + +3. **Fresh process / fresh connection.** Skipped AddTag entirely + (`--write-skip-add-tag`) and ran AddStreamedValue alone against the + already-existing sandbox tag. New native client instance, new + client-side cache, new server session. **Same error 129 — no AddS2 + bytes sent on wire.** Capture confirms 44 records ending in Close2. + +**Interpretation.** The Historian engine's runtime tag cache only +ingests tags from configured IOServers / Application Server data pipelines, +not from `HistorianAccess.AddTag`-only client flows. `HistorianAccess.AddTag` +populates `Runtime.dbo.Tag` (we confirmed wwTagKey=240 was created) but +does not register the tag with the live cache that `AddStreamedValue` +checks. That registration happens server-side when an upstream data +producer (an OPC driver, the AnE event subsystem, the Application Server +attribute store, etc.) claims the tag. + +For SDK purposes this means **`WriteValueAsync` cannot be implemented as +a generic client API against this server architecture.** The SDK's writeable +surface is realistically: + +- ✅ `EnsureTagAsync` (drives EnsT2 — 146-byte payload captured) +- ✅ `DeleteTagAsync` (drives DelT — not yet captured but should be straightforward) +- ❌ `WriteValueAsync` — won't work as designed; the server gates the + data path on tags being live in its in-memory cache +- ❓ `WriteRevisionAsync` — `HistorianAccess.AddRevisionValuesBegin/Value/End` + may use a different code path (intended for editing existing historized + data); needs a separate capture against an existing tag with stored history + +Phase 2 effective deliverables: + +- ✅ NativeTraceHarness `--scenario write` extension +- ✅ EnsT2(Float) 146-byte CTagMetadata wire bytes +- ✅ Sandbox tag `RetestSdkWriteSandbox` in Runtime DB (wwTagKey=240) +- ⏸ AddS2 — blocked architecturally; **not just a protocol gap** +- ⏸ DelT — not yet captured (need `--write-delete-after` run) +- ⏸ Revision write path — separate capture needed against a historized + tag + +## Phase 2 remaining work (revised — narrower scope) + +1. Decode the 146-byte EnsT2(Float) CTagMetadata against the IL of + `CTagUtil.ConvertTagMetadataToHistorianTag` (token `0x060055CE`), + then implement `HistorianAddTagsProtocol.SerializeAnalogCTagMetadata`. + Same approach for discrete/string variants — capture each by passing + `--write-data-type Discrete` / `String` to the harness. +2. Capture DelT wire bytes by running the harness with + `--write-delete-after`. +3. Implement public `EnsureTagAsync` + `DeleteTagAsync` only. **Drop + `WriteValueAsync` from this plan.** +4. (Stretch) probe `AddRevisionValuesBegin/Value/End` against a tag that + IS in the server cache (e.g., SysTimeSec) to see whether the revision + path bypasses the cache check. + +`WriteValueAsync` is now an OPEN QUESTION: is the only viable path for +client-driven writes the AVEVA REST API or the Application Server SDK? +File a separate plan for that investigation if SDK consumers actually +need data-write support. ## Phase 1 findings (recorded here, not implementing) diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index 7c98d16..7a2651b 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -228,6 +228,11 @@ internal static class Program } string writeDataTypeName = GetArg(args, "--write-data-type") ?? "Float"; double writeValue = double.TryParse(GetArg(args, "--write-value"), out double parsedValue) ? parsedValue : 42.5; + // --write-skip-add-tag lets the value-only second pass run without re-creating + // the sandbox. The connection's tag cache is bound at OpenConnection time, so the + // server-cache refresh after a fresh AddTag requires a NEW process / connection. + bool skipAddTag = HasFlag(args, "--write-skip-add-tag"); + bool skipAddValue = HasFlag(args, "--write-skip-add-value"); // Decoded via dnlib — actual enum field types on HistorianTag: // set_TagDataType stfld ArchestrA.HistorianDataType HistorianTag::dataType @@ -254,28 +259,34 @@ internal static class Program SetProperty(tag, "ApplyScaling", false); uint tagKey = 0; - object addError = Activator.CreateInstance(errorType)!; - MethodInfo addTagMethod = accessType.GetMethod("AddTag", new[] { tagDefType, typeof(uint).MakeByRefType(), errorType.MakeByRefType() }) - ?? throw new MissingMethodException("HistorianAccess.AddTag"); - WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tag"); - object?[] addTagArgs = [tag, tagKey, addError]; - bool addTagSuccess = (bool)addTagMethod.Invoke(access, addTagArgs)!; - tagKey = (uint)addTagArgs[1]!; - addError = addTagArgs[2]!; - snapshots["TagAfterAddTag"] = SnapshotObject(tag); - snapshots["AddTagError"] = SnapshotObject(addError); - rows.Add(new + if (!skipAddTag) { - Kind = "AddTag", - Success = addTagSuccess, - TagKey = tagKey, - ErrorDescription = GetPropertyText(addError, "ErrorDescription"), - }); + object addError = Activator.CreateInstance(errorType)!; + MethodInfo addTagMethod = accessType.GetMethod("AddTag", new[] { tagDefType, typeof(uint).MakeByRefType(), errorType.MakeByRefType() }) + ?? throw new MissingMethodException("HistorianAccess.AddTag"); + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tag"); + object?[] addTagArgs = [tag, tagKey, addError]; + bool addTagSuccess = (bool)addTagMethod.Invoke(access, addTagArgs)!; + tagKey = (uint)addTagArgs[1]!; + addError = addTagArgs[2]!; + snapshots["TagAfterAddTag"] = SnapshotObject(tag); + snapshots["AddTagError"] = SnapshotObject(addError); + rows.Add(new + { + Kind = "AddTag", + Success = addTagSuccess, + TagKey = tagKey, + ErrorDescription = GetPropertyText(addError, "ErrorDescription"), + }); + } - // If AddTag returned no key (tag already exists from a prior session), look it up. - if (tagKey == 0) + // ALWAYS look up the real wwTagKey from SQL — AddTag returns a synthetic + // placeholder key (~10000000) when the tag is freshly created, but the server + // session cache only recognizes the real Runtime.dbo.Tag.wwTagKey value + // (small int). Using the synthetic key in AddStreamedValue causes server-side + // error 168 "Tag not added to server". + using (System.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;")) { - using System.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;"); sql.Open(); using System.Data.SqlClient.SqlCommand cmd = sql.CreateCommand(); cmd.CommandText = "SELECT wwTagKey FROM Tag WHERE TagName = @t"; @@ -283,12 +294,26 @@ internal static class Program object? result = cmd.ExecuteScalar(); if (result is int existingKey) { - tagKey = (uint)existingKey; + uint realKey = (uint)existingKey; + if (realKey != tagKey) + { + rows.Add(new { Kind = "TagKeyOverride", Synthetic = tagKey, RealFromSql = realKey }); + tagKey = realKey; + } } } + // Server cache may not pick up new tags immediately. Allow a wait between AddTag + // and AddStreamedValue so the server side has time to add the new tag to its + // in-memory cache. Configurable via --write-resync-wait-seconds (default 8). + int resyncWait = int.TryParse(GetArg(args, "--write-resync-wait-seconds"), out int w) ? w : 8; + if (resyncWait > 0) + { + Thread.Sleep(TimeSpan.FromSeconds(resyncWait)); + } + // Build HistorianDataValue + push it (drives AddS2 on the wire). - if (tagKey != 0) + if (tagKey != 0 && !skipAddValue) { object value = Activator.CreateInstance(dataValueType)!; SetProperty(value, "TagKey", tagKey);