From dbb5c99c5303542ff7629b1c38e50c20d3dc2762 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 22 Jun 2026 10:41:15 -0400 Subject: [PATCH 1/2] feat(grpc-events): v6 StartEventQuery request + capture-event harness scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Captured the stock 2023 R2 client doing a gRPC event read (50 rows flowed) to resolve the open "gRPC event ROW retrieval returns zero rows" item. Two captured differences from our SDK's path; this lands the first (necessary) one plus the capture tooling. - HistorianEventQueryProtocol.CreateStartEventQueryAttempts: add a `version` parameter (default 5 = the 2020 WCF format, unchanged). The gRPC event orchestrator now opts into version 6 — the leading `06` plus a 5-byte trailing zero pad — which is the envelope the stock 2023 R2 client sends. The two buffers are otherwise byte-identical (filter block, UTC string, metadata namespace). Golden test Version6EmptyFilterMatchesCapturedGrpcEnvelope pins it. - Grpc2023CaptureHarness: new `capture-event` scenario drives HistorianAccess over an Event-type gRPC connection (CreateEventQuery -> EventQueryArgs -> StartQuery -> MoveNext) so the wide-net instrument-grpc-nonstream rewrite dumps StartEventQuery.requestBuffer + the row result. Hostname defaults sanitized to HISTORIAN_GRPC_HOST / "localhost" (removed hardcoded server name). NECESSARY BUT NOT SUFFICIENT: live validation shows v6 alone does not make rows flow — the read also requires an Event-type connection, which our SDK's v6 Open2 format cannot express (see the companion docs commit). The gated ReadEventsAsync_OverGrpc_* test correctly still pins the no-row throw. 322/322 offline tests pass; WCF event path unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../Grpc/HistorianGrpcEventOrchestrator.cs | 8 +- .../Wcf/HistorianEventQueryProtocol.cs | 30 ++- .../WcfEventQueryProtocolTests.cs | 27 +++ .../Program.cs | 186 +++++++++++++++++- 4 files changed, 238 insertions(+), 13 deletions(-) diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index 18d1b0f..ff0d62f 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -266,11 +266,17 @@ internal sealed class HistorianGrpcEventOrchestrator new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, retrievalVersion.UiVersion, _options); + // Version 6 envelope: the stock 2023 R2 client sends v6 (the WCF path's v5 request is accepted + // here but is the legacy format). NECESSARY but not alone sufficient — live validation 2026-06-22 + // showed rows still don't flow on v6 because the read also requires an EVENT-type connection + // (the stock client opens ConnectionType=Event; our OpenSession opens a Process-style 0x402 + // session). See docs/reverse-engineering/grpc-event-query-capture.md "remaining gate". IReadOnlyList attempts = HistorianEventQueryProtocol.CreateStartEventQueryAttempts( startUtc.ToUniversalTime(), endUtc.ToUniversalTime(), eventCount: 5, - filter); + filter, + version: 6); byte[] requestBuffer = attempts[0].RequestBuffer; GrpcRetrieval.StartEventQueryResponse startResponse = retrievalClient.StartEventQuery( diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianEventQueryProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianEventQueryProtocol.cs index 609fcda..8733d80 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianEventQueryProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianEventQueryProtocol.cs @@ -8,22 +8,31 @@ internal static class HistorianEventQueryProtocol { public const ushort QueryRequestTypeEvent = 3; + /// + /// Builds the StartEventQuery pRequestBuff. selects the + /// envelope revision: 5 (default) is the native 2020 WCF format used by the WCF event + /// orchestrator; 6 is the 2023 R2 gRPC format. The two envelopes are byte-identical except + /// the leading version word and a 5-byte trailing zero pad — captured 2026-06-22 from the stock + /// 2023 R2 client (see docs/reverse-engineering/grpc-event-query-capture.md). The 2023 R2 + /// server returns rows only for v6; v5 is accepted (StartEventQuery succeeds) but matches no rows. + /// The filter block in the middle is unchanged across versions. + /// public static IReadOnlyList CreateStartEventQueryAttempts( - DateTime startUtc, DateTime endUtc, uint eventCount, HistorianEventFilter? filter = null) + DateTime startUtc, DateTime endUtc, uint eventCount, HistorianEventFilter? filter = null, ushort version = 5) { List attempts = []; - attempts.Add(CreateNativeFilterAttempt(startUtc, endUtc, eventCount, filter)); + attempts.Add(CreateNativeFilterAttempt(startUtc, endUtc, eventCount, filter, version)); return attempts; } private static HistorianEventQueryAttempt CreateNativeFilterAttempt( - DateTime startUtc, DateTime endUtc, uint eventCount, HistorianEventFilter? filter) + DateTime startUtc, DateTime endUtc, uint eventCount, HistorianEventFilter? filter, ushort version) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.Unicode, leaveOpen: true); - writer.Write((ushort)5); + writer.Write(version); writer.Write(startUtc.ToFileTimeUtc()); writer.Write(endUtc.ToFileTimeUtc()); writer.Write(eventCount); @@ -43,10 +52,19 @@ internal static class HistorianEventQueryProtocol WriteMetadataNamespace(writer); writer.Write(0u); + // Version 6 (2023 R2 gRPC) appends a 5-byte trailing zero pad after the v5 terminal — the only + // envelope delta from v5 besides the version word. Captured live: the v6 buffer is the v5 buffer + // (byte 0 = 6) plus these 5 bytes, and is the form the 2023 R2 server returns event rows for. + if (version >= 6) + { + writer.Write(0u); + writer.Write((byte)0); + } + byte[] request = stream.ToArray(); return new HistorianEventQueryAttempt( - filter is null ? "native-empty-filter-version5" : "native-filter-version5", - 5, + filter is null ? $"native-empty-filter-version{version}" : $"native-filter-version{version}", + version, request, Convert.ToHexString(SHA256.HashData(request)).ToLowerInvariant()); } diff --git a/tests/AVEVA.Historian.Client.Tests/WcfEventQueryProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfEventQueryProtocolTests.cs index 5d76f8e..eab9fda 100644 --- a/tests/AVEVA.Historian.Client.Tests/WcfEventQueryProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/WcfEventQueryProtocolTests.cs @@ -73,6 +73,33 @@ public sealed class WcfEventQueryProtocolTests Assert.Equal("6b955b02087047a3199a8c74f3eee85c3b49aaa29b05de12eff2dd536f2da0d5", attempt.RequestSha256); } + [Fact] + public void Version6EmptyFilterMatchesCapturedGrpcEnvelope() + { + // Captured 2026-06-22 from the stock 2023 R2 client (docs/reverse-engineering/grpc-event-query-capture.md): + // the v6 StartEventQuery request is byte-identical to the v5 buffer except byte 0 (version 6) and a + // 5-byte trailing zero pad (70 vs 65 bytes). The 2023 R2 server returns event rows only for v6. + DateTime start = new DateTime(2026, 4, 25, 14, 39, 36, 800, DateTimeKind.Utc).AddTicks(1646); + DateTime end = new DateTime(2026, 5, 2, 14, 39, 36, 800, DateTimeKind.Utc).AddTicks(1646); + + byte[] v5 = HistorianEventQueryProtocol.CreateStartEventQueryAttempts(start, end, 3)[0].RequestBuffer; + HistorianEventQueryAttempt v6Attempt = Assert.Single( + HistorianEventQueryProtocol.CreateStartEventQueryAttempts(start, end, 3, filter: null, version: 6)); + byte[] v6 = v6Attempt.RequestBuffer; + + Assert.Equal("native-empty-filter-version6", v6Attempt.Name); + Assert.Equal(6, v6Attempt.Version); + Assert.Equal(70, v6.Length); + Assert.Equal([0x06, 0x00], v6[..2]); + + // v6 == v5 with byte 0 -> 6 and 5 trailing zero bytes appended. + byte[] expected = new byte[70]; + Array.Copy(v5, expected, v5.Length); + expected[0] = 0x06; + Assert.Equal(expected, v6); + Assert.Equal([0x00, 0x00, 0x00, 0x00, 0x00], v6[^5..]); + } + [Fact] public void NativeEmptyFilterAttemptMatchesDecompiledSaveOrder() { diff --git a/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs b/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs index 240cfe4..323c465 100644 --- a/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs +++ b/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs @@ -90,8 +90,10 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness return CaptureWrite(managedDll, args); case "delete-tag": return DeleteTag(managedDll, args); + case "capture-event": + return CaptureEvent(managedDll, args); default: - Console.Error.WriteLine($"Unknown scenario '{scenario}'. Supported: load-check, connect, capture-write, delete-tag."); + Console.Error.WriteLine($"Unknown scenario '{scenario}'. Supported: load-check, connect, capture-write, delete-tag, capture-event."); return 1; } } @@ -114,7 +116,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness Type tagStatusType = Req(asm, "ArchestrA.HistorianTagStatus"); Type tagStatusListType = Req(asm, "ArchestrA.HistorianTagStatusList"); - string server = GetOption(args, "--server") ?? "WONDER-SQL-VD03"; + string server = GetOption(args, "--server") ?? Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST") ?? "localhost"; int port = int.TryParse(GetOption(args, "--port"), out int p) ? p : 32565; string certName = GetOption(args, "--cert") ?? server; string tagName = GetOption(args, "--tag") ?? "SdkM3CaptureSandbox"; @@ -201,7 +203,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness /// SendValues (the actual wire push) only runs with --commit. Run with --grpc-rewrite pointing /// at the instrumented copy and AVEVA_HISTORIAN_RE_CAPTURE set to the output file. /// Usage: capture-write --tag SdkM3CaptureSandbox [--create] [--commit] - /// [--server WONDER-SQL-VD03] [--port 32565] [--cert WONDER-SQL-VD03] [--value 123.0] + /// [--server ] [--port 32565] [--cert ] [--value 123.0] /// private static int CaptureWrite(string managedDll, string[] args) { @@ -221,7 +223,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness Type listType = Req(asm, "ArchestrA.HistorianDataValueList"); Type categoryEnum = Req(asm, "ArchestrA.HistorianDataCategory"); - string server = GetOption(args, "--server") ?? "WONDER-SQL-VD03"; + string server = GetOption(args, "--server") ?? Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST") ?? "localhost"; int port = int.TryParse(GetOption(args, "--port"), out int p) ? p : 32565; string certName = GetOption(args, "--cert") ?? server; string tagName = GetOption(args, "--tag") ?? "SdkM3CaptureSandbox"; @@ -429,12 +431,184 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness result = m.Invoke(target, a); } + /// + /// Drives the native 2023 R2 client through a read-only gRPC EVENT query so the IL-rewritten + /// GrpcRetrievalClient dumps the uncaptured event buffers: StartEventQuery.requestBuffer (the + /// empty-filter request shape our SDK's CreateNativeEmptyFilterAttempt is being compared against) + /// and GetNextEventQueryResultBuffer.result (the row buffer — proves rows flow when driven right). + /// + /// CRITICAL: the connection is opened with ConnectionType=Event (NOT Process). CreateEventQuery() + /// returns null unless IsEventConnectionRequested() — the native event read runs on ConnectionIndex 1, + /// a separate connection from the process/data path. This is the prime suspect for why the SDK's + /// gRPC empty-filter query returns zero rows despite the server holding events. + /// + /// Sequence: OpenConnection(Event, read-only, gRPC) -> CreateEventQuery() -> + /// EventQueryArgs{StartDateTime,EndDateTime,EventCount} -> EventQuery.StartQuery(args) -> + /// loop EventQuery.MoveNext()/QueryResult -> EventQuery.EndQuery() -> CloseConnection. + /// Run with --grpc-rewrite pointing at the instrumented Archestra.Historian.GrpcClient.dll and + /// AVEVA_HISTORIAN_RE_CAPTURE set to the output NDJSON. Read-only — non-destructive. + /// Usage: capture-event [--server ] [--port 32565] [--cert ] + /// [--lookback-hours 720] [--max-events 50] [--integrated] + /// + private static int CaptureEvent(string managedDll, string[] args) + { + Assembly asm = Assembly.LoadFrom(managedDll); + Type accessType = Req(asm, "ArchestrA.HistorianAccess"); + Type connArgsType = Req(asm, "ArchestrA.HistorianConnectionArgs"); + Type connModeType = Req(asm, "ArchestrA.HistorianConnectionMode"); + Type connTypeType = Req(asm, "ArchestrA.HistorianConnectionType"); + Type errorType = Req(asm, "ArchestrA.HistorianAccessError"); + Type statusType = Req(asm, "ArchestrA.HistorianConnectionStatus"); + Type certInfoType = Req(asm, "ArchestrA.CertificateInfo"); + Type secModeType = Req(asm, "ArchestrA.HistorianSecurityMode"); + Type eventQueryType = Req(asm, "ArchestrA.EventQuery"); + Type eventArgsType = Req(asm, "ArchestrA.EventQueryArgs"); + + string server = GetOption(args, "--server") ?? Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST") ?? "localhost"; + int port = int.TryParse(GetOption(args, "--port"), out int p) ? p : 32565; + string certName = GetOption(args, "--cert") ?? server; + int lookbackHours = int.TryParse(GetOption(args, "--lookback-hours"), out int lh) ? lh : 720; + int maxEvents = int.TryParse(GetOption(args, "--max-events"), out int me) ? me : 50; + bool integrated = args.Contains("--integrated"); + string? user = Environment.GetEnvironmentVariable("HISTORIAN_USER"); + string? password = Environment.GetEnvironmentVariable("HISTORIAN_PASSWORD"); + if (!integrated && string.IsNullOrEmpty(user)) + { + Console.Error.WriteLine("Set HISTORIAN_USER/HISTORIAN_PASSWORD or pass --integrated."); + return 1; + } + + object connArgs = Activator.CreateInstance(connArgsType)!; + SetProp(connArgs, "ServerName", server); + SetProp(connArgs, "TcpPort", checked((ushort)port)); + SetProp(connArgs, "ConnectionMode", Enum.Parse(connModeType, "Historian")); // 2 = gRPC + SetProp(connArgs, "ConnectionType", Enum.Parse(connTypeType, "Event")); // EVENT connection + SetProp(connArgs, "ReadOnly", true); + SetProp(connArgs, "IntegratedSecurity", integrated); + SetProp(connArgs, "AllowUnTrustedConnection", true); + if (!integrated) + { + SetProp(connArgs, "UserName", user!); + SetProp(connArgs, "Password", password ?? string.Empty); + } + object certInfo = Activator.CreateInstance(certInfoType)!; + TrySetProp(certInfo, "CertificateName", certName); + TrySetProp(certInfo, "SecurityMode", Enum.Parse(secModeType, "TransportCertificate")); + TrySetProp(connArgs, "SecurityInfo", certInfo); + + object access = Activator.CreateInstance(accessType)!; + object error = Activator.CreateInstance(errorType)!; + object?[] openArgs = { connArgs, error }; + Console.WriteLine($"OpenConnection: server={server} port={port} mode=Historian type=Event cert={certName} integrated={integrated} readonly=true"); + bool opened; + try + { + opened = (bool)accessType.GetMethod("OpenConnection", new[] { connArgsType, errorType.MakeByRefType() })! + .Invoke(access, openArgs)!; + } + catch (TargetInvocationException tie) + { + Console.Error.WriteLine($"OpenConnection threw: {tie.InnerException?.GetType().Name}: {tie.InnerException?.Message}"); + return 2; + } + Console.WriteLine($"OpenConnection returned: {opened} err={DescribeError(openArgs[1])}"); + if (!opened) { return 2; } + + try + { + // Let the event connection (ConnectionIndex 1) come up. + MethodInfo getStatus = accessType.GetMethod("GetConnectionStatus", new[] { statusType.MakeByRefType() }) + ?? accessType.GetMethods().First(m => m.Name == "GetConnectionStatus" && m.GetParameters().Length == 1); + for (int i = 0; i < 10; i++) + { + object?[] sArgs = { null }; + getStatus.Invoke(access, sArgs); + if (ReadBoolProp(sArgs[0], "ConnectedToServer") || !ReadBoolProp(sArgs[0], "Pending")) break; + System.Threading.Thread.Sleep(500); + } + + // CreateEventQuery() is non-null only when the connection is an event connection. + MethodInfo createEventQuery = accessType.GetMethod("CreateEventQuery", Type.EmptyTypes) + ?? accessType.GetMethods().First(m => m.Name == "CreateEventQuery" && m.GetParameters().Length == 0); + object? eventQuery = createEventQuery.Invoke(access, null); + Console.WriteLine($"CreateEventQuery: {(eventQuery == null ? "NULL (event connection not established!)" : "ok")}"); + if (eventQuery == null) { return 3; } + + // Build EventQueryArgs over the populated window. Times in UTC. + object eventArgs = Activator.CreateInstance(eventArgsType)!; + DateTime endUtc = DateTime.UtcNow; + DateTime startUtc = endUtc.AddHours(-lookbackHours); + TrySetProp(eventArgs, "StartDateTime", DateTime.SpecifyKind(startUtc, DateTimeKind.Utc)); + TrySetProp(eventArgs, "EndDateTime", DateTime.SpecifyKind(endUtc, DateTimeKind.Utc)); + TrySetProp(eventArgs, "EventCount", checked((uint)maxEvents)); + Console.WriteLine($"EventQueryArgs: start={startUtc:o} end={endUtc:o} eventCount={maxEvents}"); + + // StartQuery -> triggers GrpcRetrievalClient.StartEventQuery (requestBuffer CAPTURED). + MethodInfo startQuery = eventQueryType.GetMethods() + .First(m => m.Name == "StartQuery" && m.GetParameters().Length == 2); + object?[] startArgs = { eventArgs, Activator.CreateInstance(errorType) }; + bool started = (bool)startQuery.Invoke(eventQuery, startArgs)!; + Console.WriteLine($"StartQuery: {started} err={DescribeError(startArgs[1])}"); + + // Poll rows -> triggers GetNextEventQueryResultBuffer (result buffer CAPTURED). + MethodInfo moveNext = eventQueryType.GetMethods() + .First(m => m.Name == "MoveNext" && m.GetParameters().Length == 1); + PropertyInfo? queryResult = eventQueryType.GetProperty("QueryResult"); + int rows = 0; + while (rows < maxEvents) + { + object?[] mnArgs = { Activator.CreateInstance(errorType) }; + bool more; + try { more = (bool)moveNext.Invoke(eventQuery, mnArgs)!; } + catch (TargetInvocationException tie) + { + Console.WriteLine($"MoveNext threw: {tie.InnerException?.GetType().Name}: {tie.InnerException?.Message}"); + break; + } + if (!more) + { + Console.WriteLine($"MoveNext: end after {rows} row(s) err={DescribeError(mnArgs[0])}"); + break; + } + rows++; + if (rows <= 3 && queryResult != null) + { + // Print only the event TYPE + time (non-identity) to confirm rows flow. + object? res = queryResult.GetValue(eventQuery); + string typ = res?.GetType().GetProperty("Type")?.GetValue(res)?.ToString() ?? "?"; + object? t = res?.GetType().GetProperty("EventTime")?.GetValue(res); + Console.WriteLine($" row {rows}: Type={typ} EventTime={t}"); + } + } + Console.WriteLine($"Rows iterated: {rows}"); + + MethodInfo? endQuery = eventQueryType.GetMethods() + .FirstOrDefault(m => m.Name == "EndQuery" && m.GetParameters().Length == 1); + if (endQuery != null) + { + object?[] eqArgs = { Activator.CreateInstance(errorType) }; + endQuery.Invoke(eventQuery, eqArgs); + } + Console.WriteLine(rows > 0 ? "CAPTURE-EVENT: PASS (rows flowed)" : "CAPTURE-EVENT: request captured (zero rows)"); + return 0; + } + finally + { + try + { + MethodInfo? close = accessType.GetMethod("CloseConnection", new[] { errorType.MakeByRefType() }); + if (close != null) close.Invoke(access, new object?[] { Activator.CreateInstance(errorType) }); + } + catch { /* best-effort */ } + } + } + /// /// Read-only gRPC connect probe: opens a 2023 R2 Historian (mode=Historian) connection via the /// native client and reports the resulting connection status. Proves the mixed-mode client can /// reach the live server over gRPC from this box — the foundation for the write-capture step. /// Reads creds from HISTORIAN_USER / HISTORIAN_PASSWORD (explicit) or uses IntegratedSecurity. - /// Usage: connect --server WONDER-SQL-VD03 [--port 32565] [--cert WONDER-SQL-VD03] [--integrated] + /// Usage: connect --server [--port 32565] [--cert ] [--integrated] /// private static int Connect(string managedDll, string[] args) { @@ -448,7 +622,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness Type certInfoType = Req(asm, "ArchestrA.CertificateInfo"); Type secModeType = Req(asm, "ArchestrA.HistorianSecurityMode"); - string server = GetOption(args, "--server") ?? "WONDER-SQL-VD03"; + string server = GetOption(args, "--server") ?? Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST") ?? "localhost"; int port = int.TryParse(GetOption(args, "--port"), out int p) ? p : 32565; string certName = GetOption(args, "--cert") ?? server; bool integrated = args.Contains("--integrated"); From c6752804ee58d3b743dd198d8276211f2e0369dc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 22 Jun 2026 10:41:29 -0400 Subject: [PATCH 2/2] docs(grpc-events): event-query capture finding + v8 connection-type gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the 2026-06-22 capture of the stock 2023 R2 gRPC event read and the diagnosis of why row retrieval is gated: 1. The working StartEventQuery request is version 6 (vs the SDK's v5) — shipped in the companion code commit. 2. Rows additionally require an EVENT-type connection. Decoding the captured OpenConnection.openParameters (native format v8) shows a ConnectionType byte (Event=01 / Process=02) right after ClientType — a field the SDK's v6 Open2 format does not have (it writes ClientType then ConnectionMode back-to-back). So the v6 buffer the SDK sends (accepted for reads) cannot mark the connection as Event, and the 2023 R2 server returns event rows only on an Event connection. The native client also used the ExchangeKey cert auth path. Conclusion: making event rows flow over gRPC requires the SDK to emit the native v8 OpenConnection format with ConnectionType=Event (a larger RE+implementation effort), scoped as a follow-on. v6 is retained as the captured-correct request. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/reverse-engineering/grpc-event-query-capture.md diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md new file mode 100644 index 0000000..17de437 --- /dev/null +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -0,0 +1,141 @@ +# gRPC event-query capture (2026-06-22) — the StartEventQuery request that returns rows + +Captured the stock 2023 R2 client performing a **gRPC event read** that returns rows, to resolve +the open item "gRPC event ROW retrieval returns zero rows" (handoff §Current Status item 1). This +closes the capture-gate: the working request shape is now known. + +## How it was captured + +`tools/AVEVA.Historian.Grpc2023CaptureHarness` gained a `capture-event` scenario. It loads the +self-contained mixed-mode 2023 R2 `aahClientManaged.dll` and drives `HistorianAccess`: + +``` +OpenConnection(ConnectionMode=Historian /*gRPC*/, ConnectionType=Event, ReadOnly=true) + -> CreateEventQuery() // NON-null only on an Event connection + -> EventQueryArgs { StartDateTime, EndDateTime, EventCount } + -> EventQuery.StartQuery(args) // => GrpcRetrievalClient.StartEventQuery(requestBuffer) + -> loop EventQuery.MoveNext() / QueryResult// => GrpcRetrievalClient.GetNextEventQueryResultBuffer + -> EventQuery.EndQuery() -> CloseConnection +``` + +The existing wide-net `instrument-grpc-nonstream` IL rewrite (every `Grpc*Client` `byte[]` method) +already covers `GrpcRetrievalClient.StartEventQuery.requestBuffer` (entry) and +`GetNextEventQueryResultBuffer.result` (exit) — no new instrument command was needed. Run read-only +(non-destructive) against the live 2023 R2 server over the loopback tunnel; the rewrite + capture +NDJSON stay under `artifacts/reverse-engineering/grpc-event-capture/` (gitignored — the result +buffer carries event identity data). + +Result: **50 events returned over gRPC** (Alarm.Set / Alarm.Clear rows), proving the path works when +driven through an Event connection. + +## Two findings + +### 1. The event read needs an **Event-type connection** (`ConnectionIndex 1`) + +`HistorianAccess.CreateEventQuery()` returns `null` unless `IsEventConnectionRequested()` — i.e. the +connection was opened with `ConnectionType=Event`, which the native client routes to a *separate* +connection (ConnectionIndex 1) from the process/data path. The full captured pre-query sequence on +that connection: `OpenConnection` → `ExchangeKey` → `UpdateClientStatus` → `RegisterTags`(CM_EVENT) → +`EnsureTags`(CM_EVENT) → `GetHistorianInfo` + 7×`GetSystemParameter` (Stat priming) → +`StartEventQuery` → `GetNextEventQueryResultBuffer` (rows) → `EndEventQuery` → `CloseConnection`. + +### 2. The working `StartEventQuery` request is **version 6**, not 5 + +Our SDK's `HistorianEventQueryProtocol.CreateNativeFilterAttempt` builds a **version-5** empty-filter +buffer; the stock 2023 R2 client sends **version 6**. Diffed byte-for-byte (same query window + +eventCount), the two buffers are **identical except**: + +- **byte 0: version `06` vs `05`** +- **5 additional trailing zero bytes** (stock = 70 bytes, SDK v5 = 65 bytes) + +The server returns rows for v6 and **zero rows for v5** (the v5 request is *accepted* — +`StartEventQuery` succeeds and yields a query handle — but `GetNextEventQueryResultBuffer` then +matches nothing). Everything else is shared: the two query-window FILETIMEs, `UInt32 eventCount`, +the `UInt32 65536` buffer hint, the `"UTC"` `HistorianString`, and the `01 01000001000001 0000` +metadata-namespace block. + +Captured v6 request layout (70 bytes; the FILETIMEs below are just the harness query window — no +identity data): + +``` +[0..1] UInt16 version = 6 // SDK currently sends 5 +[2..9] Int64 startUtc (FILETIME) +[10..17] Int64 endUtc (FILETIME) +[18..21] UInt32 eventCount +[22..25] UInt32 0 +[26..27] UInt16 0 +[28..29] UInt16 1 +[30..36] 7 bytes 0 // empty-filter block +[37..40] UInt32 65536 // buffer-size hint +[41..50] HistorianString "UTC" (UInt32 len=3 + UTF-16LE) +[51..60] 01 01 00 00 01 00 00 01 00 00 // metadata-namespace block (marker + 3 empty) +[61..69] 9 bytes 0 // terminal (SDK v5 writes only 4 here) +``` + +## Fix part 1 — v6 request (DONE, necessary) + +`HistorianEventQueryProtocol.CreateStartEventQueryAttempts` gained a `version` parameter (default 5 = +WCF/2020; the gRPC orchestrator passes 6). v6 emits the leading `06` and the 5-byte trailing pad. The +WCF path is unchanged (v5). Golden test `Version6EmptyFilterMatchesCapturedGrpcEnvelope` pins the +envelope; 322/322 offline tests pass. + +## Fix part 2 — EVENT connection (the remaining gate, NOT yet implemented) + +Live validation 2026-06-22: with the orchestrator now sending v6 against the event-bearing live +server, `GetNextEventQueryResultBuffer` **still long-polls and returns zero rows** (the gated test +still throws). So **v6 is necessary but not sufficient** — the read also requires an **Event-type +connection**, which our SDK does not open. + +Isolated by diffing the captured `OpenConnection.openParameters` (302 bytes, native format v8) for a +**Process** connection (`connect` scenario) vs the **Event** connection (`capture-event`): aside from +the per-session auth GUID/credential-hash regions ([22..37], [68..93], which vary between any two +sessions), the connection differs in **two clean structural bytes**: + +| offset | Process | Event | +|--------|---------|-------| +| 95 | `02` | `01` | +| 96 | `00` | `01` | + +These correspond to `HistorianConnectionType` (Process vs Event; the native event path runs on +`ConnectionIndex 1`). The problem: our SDK opens the session with the **2020 OpenConnection3 v6** +buffer (`HistorianNativeHandshake.BuildOpenConnection3Request`, `connectionMode 0x402`), which the +2023 R2 server accepts for reads but which carries no event-connection-type marker. `connectionMode` +is NOT the discriminator (2020 WCF event reads work with `0x402`); the native client distinguishes +event vs process via this separate `ConnectionType` field in its v8 `openParameters`. + +### Diagnosis (2026-06-22): the v6 Open2 format cannot express an event connection + +Decoded the native `openParameters` (302 bytes): **byte 0 = `08` (format version 8)**, then a +context GUID, username, a 26-byte session-derived region ([68..93]), machine/client-node/datasource +strings, and at **[94] `ClientType=04`** immediately followed by **[95] `ConnectionType` +(`01`=Event / `02`=Process)** + **[96] a flag (`01`/`00`)**, then the rest. + +Our SDK builds the **v6** buffer (`HistorianOpen2Protocol.SerializeNativeOpenConnection3Version6`, +byte 0 = `06`): it writes `ClientType` (1 byte) **immediately followed by `ConnectionMode` (uint)** — +there is **no `ConnectionType` byte at all**. The v8 format *inserts* `ConnectionType` (+flag) between +`ClientType` and the rest. So the v6 buffer the SDK sends (accepted by the 2023 R2 server for *reads*) +structurally cannot mark the connection as Event, and the server returns event rows only for an Event +connection. + +Two further obstacles to simply emitting v8: +- the native client authenticated via **`ExchangeKey`** (cert path; 72-byte `btInput`/`btOutput` in + the capture) whereas the SDK's gRPC handshake uses **`ValidateClientCredential`** (Negotiate). The + v8 `openParameters` [68..93] region is session-derived and tied to that auth flow. +- `ConnectionMode` is NOT the lever (2020 WCF event reads work at `0x402`); `ConnectionType` is a + distinct field that only exists from format v8. + +Also confirmed a secondary format gap: the native gRPC `EnsureTags` CM_EVENT payload is **86 bytes** +vs the SDK's `SerializeCmEventCTagMetadata` **83 bytes** (a 3-byte 2023 R2 bump, parallel to the +event-query v5→v6). This is likely benign on its own (CM_EVENT pre-exists; 2020 EnsT2 returns +benign-false yet events flow) but should be matched if the event open is ever rebuilt. + +**Conclusion — the event-connection gate is NOT a tweak.** Making event rows flow over gRPC requires +the SDK to emit the native **v8 `OpenConnection` format** with `ConnectionType=Event` (a 302-byte +buffer whose layout differs from the v6 buffer and includes a session-derived auth region), and +likely to adopt the `ExchangeKey` cert auth path. That is a substantial RE+implementation effort +comparable to the original Open2 work — scoped as a follow-on, not a quick fix. Until then the gated +`ReadEventsAsync_OverGrpc_*` test correctly still pins the no-row throw, and **v6 (part 1) is retained +as the captured-correct request format** for when the open is rebuilt. + +Capture artifacts (gitignored): `artifacts/reverse-engineering/grpc-event-capture/` — +`event-capture.ndjson` (Event), `process-connect-2.ndjson` (Process).