diff --git a/docs/plans/write-commands-reverse-engineering.md b/docs/plans/write-commands-reverse-engineering.md index 0971377..2dab732 100644 --- a/docs/plans/write-commands-reverse-engineering.md +++ b/docs/plans/write-commands-reverse-engineering.md @@ -1,7 +1,104 @@ # Plan: Reverse-Engineering Write Commands -Status: **PHASE 1 EXECUTED on 2026-05-04 — discovery complete, awaiting -operator decision on Phase 2.** No code changes; no DB writes. +Status: **PHASE 2 PARTIALLY EXECUTED on 2026-05-04** — write-scenario +harness extension built and captured the full EnsT2(Float) wire byte +sequence against a real sandbox tag. AddS2 is blocked client-side by +"Tag not added to server" (error 168) — the native `AddStreamedValue` +refuses to send because the tag isn't in the server's session cache, +even though `EnsT2` created it in the Runtime DB. AddS2 wire bytes +**not yet captured**; needs a separate session to resolve the +post-EnsT2 registration prereq (likely RTag2 with the analog tag GUID, +mirroring the event flow's RTag2(CmEventTagId)). + +## Phase 2 results + +**Sandbox tag created** in Runtime DB: `RetestSdkWriteSandbox`, +wwTagKey=240, DateCreated=2026-05-04 07:49:50. Single dedicated tag +per safety §1; no other tags touched. + +**`tools/AVEVA.Historian.NativeTraceHarness/Program.cs` extended** with +`--scenario write`: + +- New args: `--write-sandbox-tag ` (default + `RetestSdkWriteSandbox`; refuses any name that doesn't start with + `RetestSdkWrite`), `--write-value ` (default 42.5), + `--write-data-type ` (default Float), `--write-delete-after` + (best-effort cleanup). +- Toggles `ConnectionArgs.ReadOnly` to false when scenario is `write` + (otherwise the connection rejects writes with error 132 "Operation + is not enabled"). +- Calls `ArchestrA.HistorianAccess.AddTag` (drives `EnsT2` on the wire), + then `ArchestrA.HistorianAccess.AddStreamedValue` (would drive + `AddS2` but currently aborts client-side at error 168). +- Resolves the actual `wwTagKey` via SQL when `AddTag` returns 0 + because the tag already exists from a prior session. +- Public `AddStreamedValue` overload selector: instance method whose + signature is `(HistorianDataValue, …, HistorianAccessError&)` — + picks the simplest dispatcher that's actually reflectable (the + 4-param impl is private and not visible to reflection). + +**Captures landed** at +`artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/bothmessage-write-capture-latest.ndjson` +(46 records: 23 outgoing + 23 incoming). Same priming chain as the +event flow: + +``` +Hist.GetV → Hist.GetI ×2 → Hist.ValCl ×2 → Hist.Open2 → +Stat.GetV ×2 → Stat.GETHI ×2 → Hist.UpdC3 → +Stat.GetSystemParameter ×7 → Trx.GetV → Stat.GetV → Retr.GetV → +Hist.EnsT2(Float) → Hist.Close2 +``` + +No `RTag2`. The chain identical to the event flow except the EnsT2 +payload is the analog CTagMetadata instead of the event one, and there +is NO RTag2 between Open2 and EnsT2 (events used RTag2 to register +`CmEventTagId`). + +**Native EnsT2(Float) request body** (record 42, 322 bytes total; the +146-byte CTagMetadata `InBuff` payload is the new evidence target): + +```text +67 03 00 01 00 00 00 04 C6 02 01 00 00 00 00 00 +00 00 00 00 00 00 00 00 00 00 00 09 15 00 52 65 +74 65 73 74 53 64 6B 57 72 69 74 65 53 61 6E 64 +62 6F 78 FF FF FF FF FF FF FF FF FF FF FF FF FF +FF FF FF 09 18 00 53 44 4B 20 77 72 69 74 65 2D +52 45 20 73 61 6E 64 62 6F 78 20 74 61 67 09 04 +00 4D 44 41 53 02 01 01 00 00 00 01 E8 03 00 00 +D6 00 0E 4F BC DB DC 01 1A 03 09 04 00 74 65 73 +74 10 27 00 00 00 00 00 00 00 00 F0 3F FE 00 01 +01 01 +``` + +Visible fields (still being decoded against the +`CTagUtil.ConvertTagMetadataToHistorianTag` IL at token `0x060055CE`): + +- `09 15 00 RetestSdkWriteSandbox` (compact ASCII tag name, len 21) +- 16 bytes of `FF` — possibly a placeholder/sentinel for `CommonArchestraEventTypeId`-equivalent that's not used for analog +- `09 18 00 SDK write-RE sandbox tag` (compact ASCII description, len 24) +- `09 04 00 MDAS` (compact ASCII metadata provider) +- `09 04 00 test` (compact ASCII engineering unit) +- `0E 4F BC DB DC 01 1A 03` byte-pattern looks like an Int64 FILETIME (date-created ~2026) +- `10 27 00 00` = uint32 0x2710 = 10000 (storage-related) +- `00 00 00 00 00 00 F0 3F` = double 1.0 (likely IntegralDivisor or similar scaling) +- `FE 00 01 01 01` = trailer (matches event tag's `2F 27 01 01 01` shape) + +**Decoder script** at `scripts/decode-write-capture.py` for the next +session. + +## Phase 2 remaining work + +1. Decode the AddS2 prereq — find what RegisterTag / AddTagPair call + the server expects between EnsT2 and AddS2. Likely + `aahClientCommon.CHistStorage.AddTagidPairs` (token `0x0600202F`) + or `AddTagsWithServerTagId` (token `0x06002026`). +2. Capture AddS2 wire bytes once the prereq is satisfied. +3. Implement `HistorianAddTagsProtocol.SerializeAnalogCTagMetadata` / + discrete / string variants from the 146-byte capture above. +4. Implement `HistorianAddStreamValuesProtocol.Serialize` from the + yet-to-capture AddS2 bytes. +5. Implement the public surface: `EnsureTagAsync`, `WriteValueAsync`, + `DeleteTagAsync` (golden-byte + gated live integration tests). ## Phase 1 findings (recorded here, not implementing) diff --git a/scripts/decode-write-capture.py b/scripts/decode-write-capture.py new file mode 100644 index 0000000..a3765d5 --- /dev/null +++ b/scripts/decode-write-capture.py @@ -0,0 +1,79 @@ +"""Decoder for the write-flow capture at +artifacts/reverse-engineering/instrumented-wcf-writemessage-writes/ +bothmessage-write-capture-latest.ndjson. + +Lists the 46-record write-flow inventory and prints the EnsT2(analog) +CTagMetadata payload byte-by-byte for layout decoding. Output stays +commit-safe — the only string values printed are the sandbox tag name, +description, and engineering unit, all of which were chosen for the +sandbox itself (not customer data). +""" +import base64 +import json +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CAPTURE = REPO_ROOT / "artifacts" / "reverse-engineering" / "instrumented-wcf-writemessage-writes" / "bothmessage-write-capture-latest.ndjson" + +ACTION_RE = re.compile(rb"aa/(?:Hist|Retr|Trx|Stat|Stor)/[A-Za-z0-9]+(?:Response)?") + + +def main() -> int: + if not CAPTURE.exists(): + print(f"Capture file not found: {CAPTURE}") + return 1 + + with CAPTURE.open(encoding="utf-8-sig") as fh: + records = [json.loads(line) for line in fh if line.strip()] + + print(f"Total records: {len(records)}") + print() + print(f"{'#':>3} {'Phase':<6} {'Length':>6} {'Action':<48}") + print("-" * 70) + for idx, rec in enumerate(records): + body = base64.b64decode(rec["Base64"]) + match = ACTION_RE.search(body) + action = match.group(0).decode() if match else "" + phase = "Write" if rec["Phase"] == "WCF.WriteMessage.Body" else "Read" + print(f"{idx:>3} {phase:<6} {rec['Length']:>6} {action:<48}") + + # Find the EnsT2 outgoing record and dump its InBuff payload. + print() + print("== EnsT2(Float) CTagMetadata payload ==") + for idx, rec in enumerate(records): + if rec["Phase"] != "WCF.WriteMessage.Body": + continue + body = base64.b64decode(rec["Base64"]) + if b"aa/Hist/EnsT2" not in body: + continue + i = body.find(b"InBuff") + marker = body[i + 6] + if marker == 0x9F: + length = int.from_bytes(body[i + 7:i + 9], "little") + payload = body[i + 9:i + 9 + length] + elif marker == 0x9E: + length = body[i + 7] + payload = body[i + 8:i + 8 + length] + elif marker == 0xA0: + length = int.from_bytes(body[i + 7:i + 9], "little") + payload = body[i + 9:i + 9 + length + 1] + else: + print(f" Unknown marker 0x{marker:02X}; cannot extract") + return 2 + + print(f" Record {idx}: {len(payload)} bytes") + for off in range(0, len(payload), 16): + chunk = payload[off:off + 16] + hp = " ".join(f"{c:02X}" for c in chunk) + ap = "".join(chr(c) if 32 <= c < 127 else "." for c in chunk) + print(f" {off:04X} {hp:<48} |{ap}|") + return 0 + + print(" (No EnsT2 record found.)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs index 084103d..7c98d16 100644 --- a/tools/AVEVA.Historian.NativeTraceHarness/Program.cs +++ b/tools/AVEVA.Historian.NativeTraceHarness/Program.cs @@ -93,7 +93,7 @@ internal static class Program object connectionArgs = Activator.CreateInstance(connectionArgsType)!; SetProperty(connectionArgs, "ServerName", serverName); SetProperty(connectionArgs, "TcpPort", checked((ushort)tcpPort)); - SetProperty(connectionArgs, "ReadOnly", true); + SetProperty(connectionArgs, "ReadOnly", !IsWriteScenario(scenario)); SetProperty(connectionArgs, "IntegratedSecurity", integratedSecurity); SetProperty(connectionArgs, "ConnectionType", Enum.Parse(connectionType, IsEventScenario(scenario) ? "Event" : "Process")); if (directConnection) @@ -216,6 +216,152 @@ internal static class Program disposableQuery.Dispose(); } } + else if (openSuccess && status.ConnectedToServer && IsWriteScenario(scenario)) + { + // Per docs/plans/write-commands-reverse-engineering.md safety §1, refuse to run + // unless the sandbox tag name is whitelisted. + string sandboxTag = GetArg(args, "--write-sandbox-tag") ?? "RetestSdkWriteSandbox"; + if (!sandboxTag.StartsWith("RetestSdkWrite", StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "Write scenario refuses to run against tags whose name doesn't start with 'RetestSdkWrite'. Pass --write-sandbox-tag RetestSdkWriteSandbox."); + } + string writeDataTypeName = GetArg(args, "--write-data-type") ?? "Float"; + double writeValue = double.TryParse(GetArg(args, "--write-value"), out double parsedValue) ? parsedValue : 42.5; + + // Decoded via dnlib — actual enum field types on HistorianTag: + // set_TagDataType stfld ArchestrA.HistorianDataType HistorianTag::dataType + // set_TagStorageType stfld ArchestrA.HistorianStorageType HistorianTag::tagStorageType + Type tagDefType = GetType(assembly, "ArchestrA.HistorianTag"); + Type tagDataTypeEnum = GetType(assembly, "ArchestrA.HistorianDataType"); + Type tagStorageTypeEnum = GetType(assembly, "ArchestrA.HistorianStorageType"); + Type dataValueType = GetType(assembly, "ArchestrA.HistorianDataValue"); + Type dataValueDataTypeEnum = GetType(assembly, "ArchestrA.HistorianDataType"); + Type connectionIndexEnum = GetType(assembly, "ArchestrA.ConnectionIndex"); + + // Build HistorianTag for the sandbox. + object tag = Activator.CreateInstance(tagDefType)!; + SetProperty(tag, "TagName", sandboxTag); + SetProperty(tag, "TagDescription", "SDK write-RE sandbox tag"); + SetProperty(tag, "EngineeringUnit", "test"); + SetProperty(tag, "TagDataType", Enum.Parse(tagDataTypeEnum, writeDataTypeName, ignoreCase: true)); + SetProperty(tag, "TagStorageType", Enum.Parse(tagStorageTypeEnum, "Cyclic", ignoreCase: true)); + SetProperty(tag, "MinEU", 0.0); + SetProperty(tag, "MaxEU", 100.0); + SetProperty(tag, "MinRaw", 0.0); + SetProperty(tag, "MaxRaw", 100.0); + SetProperty(tag, "StorageRate", 1000u); + SetProperty(tag, "ApplyScaling", false); + + uint tagKey = 0; + object addError = Activator.CreateInstance(errorType)!; + MethodInfo addTagMethod = accessType.GetMethod("AddTag", new[] { tagDefType, typeof(uint).MakeByRefType(), errorType.MakeByRefType() }) + ?? throw new MissingMethodException("HistorianAccess.AddTag"); + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-tag"); + object?[] addTagArgs = [tag, tagKey, addError]; + bool addTagSuccess = (bool)addTagMethod.Invoke(access, addTagArgs)!; + tagKey = (uint)addTagArgs[1]!; + addError = addTagArgs[2]!; + snapshots["TagAfterAddTag"] = SnapshotObject(tag); + snapshots["AddTagError"] = SnapshotObject(addError); + rows.Add(new + { + Kind = "AddTag", + Success = addTagSuccess, + TagKey = tagKey, + ErrorDescription = GetPropertyText(addError, "ErrorDescription"), + }); + + // If AddTag returned no key (tag already exists from a prior session), look it up. + if (tagKey == 0) + { + using System.Data.SqlClient.SqlConnection sql = new("Server=.;Database=Runtime;Integrated Security=SSPI;"); + sql.Open(); + using System.Data.SqlClient.SqlCommand cmd = sql.CreateCommand(); + cmd.CommandText = "SELECT wwTagKey FROM Tag WHERE TagName = @t"; + cmd.Parameters.AddWithValue("@t", sandboxTag); + object? result = cmd.ExecuteScalar(); + if (result is int existingKey) + { + tagKey = (uint)existingKey; + } + } + + // Build HistorianDataValue + push it (drives AddS2 on the wire). + if (tagKey != 0) + { + object value = Activator.CreateInstance(dataValueType)!; + SetProperty(value, "TagKey", tagKey); + SetProperty(value, "DataValueType", Enum.Parse(dataValueDataTypeEnum, "Float", ignoreCase: true)); + SetProperty(value, "OpcQuality", (ushort)192); // Good + SetProperty(value, "Value", writeValue); + SetProperty(value, "StartDateTime", DateTime.UtcNow); + SetProperty(value, "EndDateTime", DateTime.UtcNow); + SetProperty(value, "ApplyScaling", false); + + // The public AddStreamedValue overloads (per dnlib) are: + // 0x0600618C — public instance, locals(HistorianDataValue, DateTime) + // 0x0600618D — public instance, no locals (simplest dispatcher) + // The 0x0600618E impl is private and not reachable by reflection. + // Pick the public instance overload whose parameters are + // (HistorianDataValue, [bool/DateTime], HistorianAccessError&) — the + // 0x0600618D dispatcher matches 3 params. + MethodInfo addValueMethod = accessType.GetMethods() + .Where(m => m.Name == "AddStreamedValue") + .OrderBy(m => m.GetParameters().Length) + .First(m => + { + ParameterInfo[] ps = m.GetParameters(); + return ps.Length >= 2 && ps[0].ParameterType == dataValueType + && ps[ps.Length - 1].ParameterType.IsByRef + && ps[ps.Length - 1].ParameterType.GetElementType() == errorType; + }); + WriteRuntimeMethodPointerSnapshot(assembly, runtimeMethodPointerOutput, runtimeMethodPointerFilters, repoRoot, scenario, "before-add-value"); + object addValueError = Activator.CreateInstance(errorType)!; + ParameterInfo[] addValueParams = addValueMethod.GetParameters(); + object?[] addValueArgs = new object?[addValueParams.Length]; + addValueArgs[0] = value; + for (int i = 1; i < addValueParams.Length - 1; i++) + { + Type pt = addValueParams[i].ParameterType; + addValueArgs[i] = pt == typeof(bool) ? false + : pt == typeof(DateTime) ? DateTime.UtcNow + : pt.IsValueType ? Activator.CreateInstance(pt) : null; + } + addValueArgs[addValueParams.Length - 1] = addValueError; + bool addValueSuccess = (bool)addValueMethod.Invoke(access, addValueArgs)!; + addValueError = addValueArgs[addValueArgs.Length - 1]!; + snapshots["ValueAfterAddStreamedValue"] = SnapshotObject(value); + snapshots["AddValueError"] = SnapshotObject(addValueError); + rows.Add(new + { + Kind = "AddStreamedValue", + Success = addValueSuccess, + Value = writeValue, + ErrorDescription = GetPropertyText(addValueError, "ErrorDescription"), + }); + + // Optionally delete the tag for clean rollback. + if (HasFlag(args, "--write-delete-after")) + { + object deleteError = Activator.CreateInstance(errorType)!; + MethodInfo deleteMethod = accessType.GetMethods().FirstOrDefault(m => + m.Name == "DeleteTags" && m.GetParameters().Length == 2) + ?? throw new MissingMethodException("HistorianAccess.DeleteTags"); + StringCollection tagsToDelete = []; + tagsToDelete.Add(sandboxTag); + object?[] deleteArgs = [tagsToDelete, deleteError]; + bool deleteSuccess = (bool)deleteMethod.Invoke(access, deleteArgs)!; + deleteError = deleteArgs[1]!; + rows.Add(new + { + Kind = "DeleteTags", + Success = deleteSuccess, + ErrorDescription = GetPropertyText(deleteError, "ErrorDescription"), + }); + } + } + } else if (openSuccess && status.ConnectedToServer && IsTagScenario(scenario)) { object query = accessType.GetMethod("CreateTagQuery", Type.EmptyTypes)!.Invoke(access, Array.Empty())!; @@ -771,6 +917,13 @@ internal static class Program || scenario.Equals("tag-query", StringComparison.OrdinalIgnoreCase); } + private static bool IsWriteScenario(string scenario) + { + return scenario.Equals("write", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("writes", StringComparison.OrdinalIgnoreCase) + || scenario.Equals("tag-write", StringComparison.OrdinalIgnoreCase); + } + private static Dictionary SnapshotObject(object target) { Dictionary snapshot = new(StringComparer.OrdinalIgnoreCase);