From 0b1e9d0a7fce4706542b83b271213126c181a607 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 09:45:52 -0400 Subject: [PATCH 1/9] feat(grpc-events): v8 OpenConnection serializer + native error decode (Path A disproven) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the native 2023 R2 version-8 OpenConnection format, which (unlike v6) carries a ConnectionType byte (Event vs Process) — required because the 2023 R2 server returns event rows only on an Event connection. - HistorianOpen2Protocol.SerializeNativeOpenConnectionVersion8: reproduces the 302-byte v8 layout decoded from a live capture (version 8, markers, client-key GUID, username HString, length-prefixed credential token, ClientType / ConnectionType / flag / constant word / compact metadata / two empty strings; the tail reuses WriteClientCommonInfo). Golden-tested (Version8EventSerializerReproducesCapturedNativeStructure). - HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request: ConnectionType= Event, zeroed credential token (mirroring how v6 zeros its credential block and relies on the separate ValidateClientCredential handshake). - HistorianGrpcHandshake.OpenSession: optional eventConnection switch; the OpenConnection failure path now decodes the native error (type/code/ASCII). Path A (reuse ValidateClientCredential + zeroed token) was live-tested and DISPROVEN: the server parses the v8 buffer but rejects it at the auth check with native 132/34 "EstablishConnection Failed to get client key" — the v8 path looks up the client key in the registry HistoryService.ExchangeKey (ECDH) populates, not the one ValidateClientCredential does. The event orchestrator is therefore reverted to the v6 session (gated test still pins the no-row throw). The v8 serializer/builder are retained for Path B (implement ExchangeKey). 323/323 offline tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../Grpc/HistorianGrpcEventOrchestrator.cs | 5 ++ .../Grpc/HistorianGrpcHandshake.cs | 17 +++++-- .../Wcf/HistorianNativeHandshake.cs | 50 +++++++++++++++++++ .../Wcf/HistorianOpen2Protocol.cs | 43 ++++++++++++++++ .../WcfOpen2ProtocolTests.cs | 38 ++++++++++++++ 5 files changed, 149 insertions(+), 4 deletions(-) diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index ff0d62f..4c2dfc9 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -138,6 +138,11 @@ internal sealed class HistorianGrpcEventOrchestrator private List RunEventChain(DateTime startUtc, DateTime endUtc, HistorianEventFilter? filter, CancellationToken cancellationToken) { using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); + // NOTE: the event read needs an Event-type (v8) connection, but the v8 OpenConnection is + // server-coupled to HistoryService.ExchangeKey (ECDH) — it rejects a v8 open made on our + // ValidateClientCredential session with native 132/34 "EstablishConnection Failed to get client + // key" (Path A, disproven 2026-06-23). Staying on the v6 session until ExchangeKey is + // implemented (Path B). The v8 serializer + BuildEventOpenConnectionVersion8Request are ready. HistorianGrpcHandshake.Session session = HistorianGrpcHandshake.OpenSession(connection, _options, cancellationToken); RegisterCmEventTag(connection, session, cancellationToken); diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs index 79d88b1..eb0972c 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs @@ -50,7 +50,8 @@ internal static class HistorianGrpcHandshake HistorianGrpcConnection connection, HistorianClientOptions options, CancellationToken cancellationToken, - uint connectionMode = HistorianWcfAuthChainHelper.NativeIntegratedReadOnlyConnectionMode) + uint connectionMode = HistorianWcfAuthChainHelper.NativeIntegratedReadOnlyConnectionMode, + bool eventConnection = false) { DateTime Deadline() => DateTime.UtcNow.Add(options.RequestTimeout); @@ -79,8 +80,12 @@ internal static class HistorianGrpcHandshake options, cancellationToken); - byte[] open2Request = HistorianNativeHandshake.BuildOpenConnection3Request( - options.Host, contextKey, connectionMode); + // Event reads require an Event-type connection (ConnectionType=Event), which only the native + // v8 OpenConnection format carries — the v6 buffer has no such field. Reuse the same + // ValidateClientCredential session; the v8 credential token is zeroed (auth already done). + byte[] open2Request = eventConnection + ? HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request(contextKey, options.UserName) + : HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); GrpcHistory.OpenConnectionResponse open2 = historyClient.OpenConnection( new GrpcHistory.OpenConnectionRequest { BtConnectionRequest = ByteString.CopyFrom(open2Request) }, @@ -92,7 +97,11 @@ internal static class HistorianGrpcHandshake if (!(open2.Status?.BSuccess ?? false)) { byte[] err = open2.Status?.BtError?.ToByteArray() ?? []; - throw new InvalidOperationException($"gRPC OpenConnection failed (errorLen={err.Length}, responseLen={open2Response.Length})."); + HistorianNativeError? decoded = HistorianOpen2Protocol.TryReadNativeError(err); + string ascii = new(err.Where(b => b is >= 0x20 and < 0x7F).Select(b => (char)b).ToArray()); + throw new InvalidOperationException( + $"gRPC OpenConnection failed (errorLen={err.Length}, responseLen={open2Response.Length}, " + + $"native={decoded?.Type}/{decoded?.Code}{(decoded?.Name is { } n ? $" {n}" : "")}, ascii='{ascii}')."); } (uint clientHandle, Guid storageSessionId) = HistorianNativeHandshake.ParseOpenConnectionResponse(open2Response); diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs index fcec618..77d9d4c 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs @@ -134,6 +134,56 @@ internal static class HistorianNativeHandshake credentialBlock: new byte[CredentialBlockSizeBytes]); } + // v8 OpenConnection constants (2023 R2), decoded from the native Event-connection capture. + private const byte NativeConnectionTypeEvent = 1; + private const byte NativeConnectionFlagEvent = 1; + private const ushort NativeHcalVersionV8 = 18; + private const string ClientDataSourceIdV8 = "2023.1219.4004.5"; + private const byte NativeClientCommonInfoFormatVersionV8 = 3; + private const int CredentialTokenSizeBytes = 26; + private static readonly Guid UnsetShardId = new(new byte[] + { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + }); + + /// + /// Builds the native 2023 R2 version-8 OpenConnection request for an Event connection + /// (ConnectionType=Event). The 2023 R2 server returns event rows only on an event connection, + /// and the v6 OpenConnection buffer has no ConnectionType field. The credential token is zeroed: the + /// session is authenticated by the preceding ValidateClientCredential handshake, exactly as + /// the v6 path already relies on (it sends a zeroed credential block). See + /// docs/reverse-engineering/grpc-event-query-capture.md. + /// + public static byte[] BuildEventOpenConnectionVersion8Request(Guid contextKey, string userName) + { + Process current = Process.GetCurrentProcess(); + string machineName = Environment.MachineName; + string processName = string.IsNullOrEmpty(current.ProcessName) ? ClientNodeNameFallback : current.ProcessName; + + HistorianClientCommonInfo commonInfo = new( + FormatVersion: NativeClientCommonInfoFormatVersionV8, + ServerNodeName: machineName, + ClientNodeName: processName, + ProcessId: checked((uint)current.Id), + HcalVersion: NativeHcalVersionV8, + ProcessName: string.Empty, + Proxy: string.Empty, + DataSourceId: ClientDataSourceIdV8, + ShardId: UnsetShardId, + ClientVersion: NativeClientVersionInt, + ClientTimestamp: (ulong)DateTime.UtcNow.ToFileTimeUtc(), + ClientDllVersion: string.Empty); + + return HistorianOpen2Protocol.SerializeNativeOpenConnectionVersion8( + commonInfo, + contextKey, + NativeConnectionTypeEvent, + NativeConnectionFlagEvent, + userName ?? string.Empty, + credentialToken: new byte[CredentialTokenSizeBytes]); + } + /// /// Decodes the OpenConnection response blob: byte 0 = protocol version, bytes 1..4 = /// transient /Retr client handle (UInt32 LE), bytes 5..20 = storage session GUID. diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianOpen2Protocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianOpen2Protocol.cs index f60f061..5cf5831 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianOpen2Protocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianOpen2Protocol.cs @@ -50,6 +50,49 @@ internal static class HistorianOpen2Protocol return stream.ToArray(); } + /// + /// Builds the native 2023 R2 version-8 OpenConnection request — the format the stock client + /// uses, which (unlike v6) carries a byte (Event vs Process). The + /// tail is the same v6 emits; the head is the v8 layout + /// decoded from a live capture (docs/reverse-engineering/grpc-event-query-capture.md): version 8, + /// two marker bytes, the client-key GUID, a username HistorianString, a length-prefixed + /// credential token, then ClientType / ConnectionType / flag / a constant word / compact metadata + /// namespace / two empty strings. The credential token is normally an ECDH-derived blob in the + /// native cert path; this serializer sends it as supplied (zeroed for the Negotiate-auth path, + /// mirroring how the v6 request already sends a zeroed credential block and relies on the separate + /// ValidateClientCredential handshake for authentication). + /// + public static byte[] SerializeNativeOpenConnectionVersion8( + HistorianClientCommonInfo commonInfo, + Guid clientKey, + byte connectionType, + byte connectionFlag, + string userName, + byte[] credentialToken) + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream, Encoding.Unicode, leaveOpen: true); + + writer.Write((byte)8); // [0] format version 8 + writer.Write((byte)0xF0); // [1] constant marker + writer.Write(new byte[19]); // [2..20] zero padding + writer.Write((byte)1); // [21] constant marker + writer.Write(clientKey.ToByteArray()); // [22..37] per-session client key + WriteHistorianString(writer, userName); // username (UTF-16 HistorianString) + writer.Write((ushort)credentialToken.Length); // credential-token length (u16) + writer.Write(credentialToken); // credential token (zeroed for the Negotiate path) + writer.Write((byte)4); // ClientType = 4 + writer.Write(connectionType); // ConnectionType (Event = 1 / Process = 2) + writer.Write(connectionFlag); // type flag (Event = 1 / Process = 0) + writer.Write((ushort)3); // constant word observed at [97..98] + WriteCompactMetadataNamespace(writer, HistorianMetadataNamespace.Empty); + WriteHistorianString(writer, string.Empty); + WriteHistorianString(writer, string.Empty); + WriteClientCommonInfo(writer, commonInfo); // tail: FormatVersion 3 => no ClientDllVersion + writer.Write(0u); // trailing terminator + return stream.ToArray(); + } + private static void WriteNativeOpenConnectionContent( BinaryWriter writer, HistorianOpen2Request request, diff --git a/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs index 3646d94..3afd730 100644 --- a/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs @@ -5,6 +5,44 @@ namespace AVEVA.Historian.Client.Tests; public sealed class WcfOpen2ProtocolTests { + [Fact] + public void Version8EventSerializerReproducesCapturedNativeStructure() + { + // Field sizes chosen to match the live 2023 R2 Event-connection capture (302 bytes): + // username=12, token=26, serverNode=15, clientNode=32, datasource=16. + // See docs/reverse-engineering/grpc-event-query-capture.md. + var clientKey = new Guid("11223344-5566-7788-99aa-bbccddeeff00"); + byte[] actual = HistorianOpen2Protocol.SerializeNativeOpenConnectionVersion8( + new HistorianClientCommonInfo( + FormatVersion: 3, + ServerNodeName: new string('S', 15), + ClientNodeName: new string('C', 32), + ProcessId: 0x11223344, + HcalVersion: 18, + ProcessName: string.Empty, + Proxy: string.Empty, + DataSourceId: new string('D', 16), + ShardId: new Guid(Enumerable.Repeat((byte)0xFF, 16).ToArray()), + ClientVersion: 999_999, + ClientTimestamp: 0x01DD02426F9B6F6C, + ClientDllVersion: string.Empty), + clientKey, + connectionType: 1, // Event + connectionFlag: 1, // Event + userName: new string('U', 12), + credentialToken: new byte[26]); + + Assert.Equal(302, actual.Length); // exact native length for these field sizes + Assert.Equal(0x08, actual[0]); // format version 8 + Assert.Equal(0xF0, actual[1]); // marker + Assert.Equal(0x01, actual[21]); // marker + Assert.Equal(clientKey.ToByteArray(), actual[22..38]); + Assert.Equal(0x04, actual[94]); // ClientType + Assert.Equal(0x01, actual[95]); // ConnectionType = Event + Assert.Equal(0x01, actual[96]); // type flag = Event + Assert.Equal([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], actual[270..286]); // ShardId + } + [Fact] public void LegacyVersion1SerializerMatchesDecompiledSaveOpenConnectionParamsLayout() { From 7284fdc976a13c2378e746958b5f36472d7d9f36 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 09:46:07 -0400 Subject: [PATCH 2/9] =?UTF-8?q?docs(grpc-events):=20Path=20A=20disproven?= =?UTF-8?q?=20=E2=80=94=20v8=20OpenConnection=20coupled=20to=20ExchangeKey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the full v8 openParameters byte map, the ECDH ExchangeKey finding, and the Path A live result: the v8 OpenConnection on a ValidateClientCredential session is rejected with native 132/34 "EstablishConnection Failed to get client key". The v8 path requires the client key established by HistoryService.ExchangeKey (ECDH), so the next route is Path B — implement ExchangeKey ("ECK1" + 64-byte P-256 point) via .NET ECDiffieHellman, then reissue the v8 open. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md index 17de437..cb56466 100644 --- a/docs/reverse-engineering/grpc-event-query-capture.md +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -139,3 +139,93 @@ as the captured-correct request format** for when the open is rebuilt. Capture artifacts (gitignored): `artifacts/reverse-engineering/grpc-event-capture/` — `event-capture.ndjson` (Event), `process-connect-2.ndjson` (Process). + +## v8 `openParameters` fully decoded (2026-06-23) + the ECDH ExchangeKey finding + +Full byte map of the native Event-connection `openParameters` (302 bytes; identity values +redacted — they are session-specific and sit in the gitignored capture): + +``` +[0] byte 0x08 format version = 8 +[1] byte 0xf0 constant marker +[2..20] 19 × 0x00 +[21] byte 0x01 constant marker +[22..37] 16B GUID per-session client key +[38..41] u32 username length (chars) +[42..N] UTF-16 username (HistorianString) +[..+1] u16 credential-token length (= 26 in the capture) +[..] 26B token ECDH-derived credential token <-- see below +[94] byte 0x04 ClientType (= our NativeClientType 4) +[95] byte ConnectionType 01 = Event / 02 = Process <-- THE GATE +[96] byte flag 01 (Event) / 00 (Process) +[97..] control bytes (0x03 ... small region, not fully named) +[~114..117]u32 FormatVersion=3 +[..] HistorianString machine/server node name +[..] HistorianString client node name "()" +[..] u32 session-variable (process-ish) +[..] u32 / zeros +[..] u32 datasource len +[..] UTF-16 datasource id e.g. "2023.1219.4004.5" +[270..285] 16 × 0xff ShardId (all-FF = unset; our v6 sends Empty) +[286..289] u32 client/hcal version int +[290..297] i64 FILETIME ClientTimestamp +[298..301] u32 0 +``` + +The tail (`FormatVersion` → machine → clientNode → datasource → ShardId → version → timestamp) +is the **same `ClientCommonInfo` our v6 already emits**. The new/different parts are: version byte, +the `[1]`/`[21]` markers, the GUID position, the **26-byte credential token** (vs v6's fixed-size +block), the **`ConnectionType` byte**, and ShardId=FF. + +**The auth is ECDH, not Negotiate.** The capture's `ExchangeKey` buffers begin `45 43 4b 31` = +ASCII **`"ECK1"`** + a 64-byte EC public-key point — a Diffie-Hellman key exchange — and the 26-byte +`openParameters` token is derived from it. `HistorianSecurityMode` offers only `Disabled` / `None` / +`TransportCertificate`; the harness used `TransportCertificate`, which is what drives the ECDH +`ExchangeKey`. There is **no TLS+Negotiate mode** on the native client (it couples TLS with the cert +ECDH path), so a Negotiate-auth v8 capture cannot be produced from the native client. + +**Key de-risking insight:** our SDK's v6 `OpenConnection` sends a **fully zeroed** 1026-byte +credential block (`credentialBlock: new byte[1026]`) and reads still work — because authentication is +actually carried by the separate `StorageService.ValidateClientCredential` (Negotiate) handshake, not +by the bytes inside `openParameters`. By analogy the v8 `[68..93]` token may likewise be **ignorable** +once `ValidateClientCredential` has run. So the first build hypothesis (cheapest, read-only to test): + +> Reuse the SDK's existing `ValidateClientCredential` handshake, then send a **v8 `OpenConnection` +> with `ConnectionType=Event` and a zeroed credential token**, and see whether the 2023 R2 server +> returns event rows. + +If that works, the ECDH ExchangeKey RE is unnecessary. If it fails, the fallback is full reproduction +of the ECDH `ExchangeKey` handshake (curve/KDF/cipher) — a much larger crypto-RE effort. Build path: +add `SerializeNativeOpenConnectionVersion8(connectionType)` to `HistorianOpen2Protocol`, wire the gRPC +event handshake to use it (events only; reads stay on v6), live-test (non-destructive). Full hex in +the gitignored capture. + +### Path A built + live-tested 2026-06-23 — DISPROVEN (v8 is coupled to ExchangeKey) + +Built `HistorianOpen2Protocol.SerializeNativeOpenConnectionVersion8` (golden-tested, +`Version8EventSerializerReproducesCapturedNativeStructure` — reproduces the captured 302-byte +structure exactly) + `HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request` (zeroed +credential token) + an `eventConnection` switch on `HistorianGrpcHandshake.OpenSession`, and live-ran +the event read against the server. Result: the v8 `OpenConnection` was **parsed by the server** (got +past the byte format) but **rejected at the auth check** with native error + +``` +type=132 code=34 "aahHcapLib::HistoryService::EstablishConnection — Failed to get client key" +``` + +i.e. `EstablishConnection` could not find a server-side **client key** for our session. In the v6 +path that key is established by `StorageService.ValidateClientCredential` (which is why v6 reads +work); the v8 path looks it up in the registry that **`HistoryService.ExchangeKey` (ECDH)** populates, +and there is **no `ValidateClientCredential` on `HistoryService`** in the gRPC contract. So the server +branches on the OpenConnection version: v6 accepts the Negotiate-established key, **v8 requires the +ExchangeKey-established key**. The zeroed-token hypothesis is therefore disproven — not because of the +token bytes, but because the whole v8 path is gated on `ExchangeKey` having run first. + +**Status:** the v8 serializer/builder are correct and retained (golden-tested), plus the +`OpenConnection` failure now decodes the native error (type/code/ASCII). The event orchestrator is +reverted to the v6 session (gated test still pins the no-row throw). The remaining route is **Path B: +implement `HistoryService.ExchangeKey`** — `"ECK1"` + a 64-byte EC public-key point (P-256 X‖Y, by the +size) — using .NET `ECDiffieHellman`, establish the client key, then reissue the v8 `OpenConnection`. +Open question for Path B: whether merely *completing* the ECDH key agreement registers the client key +(so the zeroed openParameters token still rides through), or whether the token must also be derived +from the shared secret (full KDF/cipher RE). From d67f6f5e96749398251722af83d32edbdd9dbaec Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 10:31:37 -0400 Subject: [PATCH 3/9] =?UTF-8?q?feat(grpc-events):=20ExchangeKey=20ECDH=20(?= =?UTF-8?q?Path=20B)=20=E2=80=94=20clears=20the=20v8=20client-key=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements HistoryService.ExchangeKey as a pure-managed P-256 ECDH key exchange and wires it ahead of the v8 Event OpenConnection. - HistorianNativeHandshake.BuildExchangeKeyClientHello / DeriveExchangeKeySecret: .NET ECDiffieHellman (nistP256); wire format "ECK1" + u32(32) + X(32) + Y(32), decoded from the live capture. No native AVEVA dependency. - HistorianGrpcHandshake.OpenSession(eventConnection: true): runs ExchangeKey on the context-key handle before the v8 OpenConnection. - Guardrail HistorianGrpcHandshakeRoutingTests scoped to the token-loop closure: still pins that the Negotiate token loop routes to ValidateClientCredential (not ExchangeKey), while allowing the legitimate ExchangeKey call in OpenSession. Live result: ExchangeKey succeeds (server accepts our public key) and the v8 OpenConnection error advances from 132/34 "Failed to get client key" to 132/171 AuthenticationFailed — the ECDH cleared the client-key layer. The remaining blocker is the 26-byte v8 credential token, which must be derived from the ECDH shared secret (token KDF, not yet recovered). Orchestrator stays on v6 (set eventConnection: true to re-arm once the KDF lands). 323/323 offline. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../Grpc/HistorianGrpcEventOrchestrator.cs | 11 ++-- .../Grpc/HistorianGrpcHandshake.cs | 28 ++++++++- .../Wcf/HistorianNativeHandshake.cs | 63 +++++++++++++++++++ .../HistorianGrpcHandshakeRoutingTests.cs | 27 ++++---- 4 files changed, 111 insertions(+), 18 deletions(-) diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index 4c2dfc9..4d3ca7b 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -138,11 +138,12 @@ internal sealed class HistorianGrpcEventOrchestrator private List RunEventChain(DateTime startUtc, DateTime endUtc, HistorianEventFilter? filter, CancellationToken cancellationToken) { using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); - // NOTE: the event read needs an Event-type (v8) connection, but the v8 OpenConnection is - // server-coupled to HistoryService.ExchangeKey (ECDH) — it rejects a v8 open made on our - // ValidateClientCredential session with native 132/34 "EstablishConnection Failed to get client - // key" (Path A, disproven 2026-06-23). Staying on the v6 session until ExchangeKey is - // implemented (Path B). The v8 serializer + BuildEventOpenConnectionVersion8Request are ready. + // Event reads need an Event-type (v8) connection. Path B established the v8 ExchangeKey (ECDH) + // client key — which cleared the "Failed to get client key" check — but the v8 OpenConnection + // then fails at native 132/171 AuthenticationFailed: the 26-byte credential token must be derived + // from the ECDH shared secret (the token-KDF is not yet reverse-engineered). Staying on the v6 + // session until that derivation lands; the ExchangeKey + v8 serializer are ready (set + // eventConnection: true to re-arm). See docs/reverse-engineering/grpc-event-query-capture.md. HistorianGrpcHandshake.Session session = HistorianGrpcHandshake.OpenSession(connection, _options, cancellationToken); RegisterCmEventTag(connection, session, cancellationToken); diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs index eb0972c..4d5539d 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs @@ -1,3 +1,4 @@ +using System.Security.Cryptography; using Google.Protobuf; using Grpc.Core; using AVEVA.Historian.Client.Wcf; @@ -81,8 +82,31 @@ internal static class HistorianGrpcHandshake cancellationToken); // Event reads require an Event-type connection (ConnectionType=Event), which only the native - // v8 OpenConnection format carries — the v6 buffer has no such field. Reuse the same - // ValidateClientCredential session; the v8 credential token is zeroed (auth already done). + // v8 OpenConnection format carries — the v6 buffer has no such field. The v8 OpenConnection also + // looks up its client key in the registry HistoryService.ExchangeKey (ECDH) populates (not the + // one ValidateClientCredential does), so establish that key first via a P-256 key exchange. + if (eventConnection) + { + using ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256); + byte[] clientHello = HistorianNativeHandshake.BuildExchangeKeyClientHello(ecdh); + string xkHandle = contextKey.ToString("D").ToUpperInvariant(); + GrpcHistory.ExchangeKeyResponse xk = historyClient.ExchangeKey( + new GrpcHistory.ExchangeKeyRequest { StrHandle = xkHandle, BtInput = ByteString.CopyFrom(clientHello) }, + connection.Metadata, + Deadline(), + cancellationToken); + if (!(xk.Status?.BSuccess ?? false)) + { + byte[] xkErr = xk.Status?.BtError?.ToByteArray() ?? []; + HistorianNativeError? xkDecoded = HistorianOpen2Protocol.TryReadNativeError(xkErr); + string xkAscii = new(xkErr.Where(b => b is >= 0x20 and < 0x7F).Select(b => (char)b).ToArray()); + throw new InvalidOperationException( + $"gRPC ExchangeKey failed (errorLen={xkErr.Length}, native={xkDecoded?.Type}/{xkDecoded?.Code}, ascii='{xkAscii}')."); + } + // The raw ECDH shared secret (if the v8 credential token later needs derivation) is: + // HistorianNativeHandshake.DeriveExchangeKeySecret(ecdh, xk.BtOutput?.ToByteArray() ?? []); + } + byte[] open2Request = eventConnection ? HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request(contextKey, options.UserName) : HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs index 77d9d4c..a2c2e93 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using System.Diagnostics; +using System.Security.Cryptography; namespace AVEVA.Historian.Client.Wcf; @@ -28,6 +29,68 @@ internal static class HistorianNativeHandshake /// Result of one transport-level credential-token exchange. internal readonly record struct TokenExchangeResult(bool Success, byte[] ServerOutput, byte[] Error); + // HistoryService.ExchangeKey wire format (decoded from a live 2023 R2 capture): the ASCII magic + // "ECK1", a uint32 coordinate length (32 = P-256), then the raw EC public-key point X || Y. Used by + // the v8 (event) connection path to establish the client key the v8 OpenConnection requires. + private static readonly byte[] ExchangeKeyMagic = "ECK1"u8.ToArray(); + private const int ExchangeKeyCoordinateBytes = 32; // P-256 / secp256r1 + + /// + /// Builds the ExchangeKey client hello: "ECK1" + u32(32) + X(32) + Y(32) for the + /// supplied P-256 key. Pure managed (); no native AVEVA dependency. + /// + public static byte[] BuildExchangeKeyClientHello(ECDiffieHellman ecdh) + { + ECParameters parameters = ecdh.ExportParameters(includePrivateParameters: false); + byte[] x = LeftPadCoordinate(parameters.Q.X!); + byte[] y = LeftPadCoordinate(parameters.Q.Y!); + + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream); + writer.Write(ExchangeKeyMagic); + writer.Write((uint)ExchangeKeyCoordinateBytes); + writer.Write(x); + writer.Write(y); + return stream.ToArray(); + } + + /// + /// Parses the server's ExchangeKey hello (same "ECK1" + u32 + X + Y shape) and + /// derives the raw ECDH shared secret against . + /// + public static byte[] DeriveExchangeKeySecret(ECDiffieHellman ecdh, byte[] serverHello) + { + const int headerLength = 8; // 4-byte magic + 4-byte coordinate length + int needed = headerLength + (2 * ExchangeKeyCoordinateBytes); + if (serverHello.Length < needed) + { + throw new InvalidOperationException($"ExchangeKey server hello too short (len={serverHello.Length}, need>={needed})."); + } + + byte[] x = serverHello[headerLength..(headerLength + ExchangeKeyCoordinateBytes)]; + byte[] y = serverHello[(headerLength + ExchangeKeyCoordinateBytes)..(headerLength + (2 * ExchangeKeyCoordinateBytes))]; + + ECParameters serverParameters = new() + { + Curve = ECCurve.NamedCurves.nistP256, + Q = new ECPoint { X = x, Y = y }, + }; + using ECDiffieHellman serverKey = ECDiffieHellman.Create(serverParameters); + return ecdh.DeriveRawSecretAgreement(serverKey.PublicKey); + } + + private static byte[] LeftPadCoordinate(byte[] coordinate) + { + if (coordinate.Length == ExchangeKeyCoordinateBytes) + { + return coordinate; + } + + byte[] padded = new byte[ExchangeKeyCoordinateBytes]; + Array.Copy(coordinate, 0, padded, ExchangeKeyCoordinateBytes - coordinate.Length, coordinate.Length); + return padded; + } + /// /// Performs a single credential-token round on the wire. is the /// upper-case context-key GUID, is the AVEVA-wrapped SSPI diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs index fee9f3f..4cea64a 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs @@ -18,26 +18,31 @@ namespace AVEVA.Historian.Client.Tests; public sealed class HistorianGrpcHandshakeRoutingTests { [Fact] - public void Handshake_UsesValidateClientCredential_NotExchangeKey() + public void Handshake_TokenLoop_UsesValidateClientCredential_NotExchangeKey() { - // The auth token loop lives in the shared handshake helper (reused by the read, status, - // and future browse/metadata gRPC paths). - HashSet calledMethods = CollectCalledMethodNames( - "AVEVA.Historian.Client.Grpc.HistorianGrpcHandshake"); + // The auth token loop is a compiler-generated lambda (nested closure) passed to RunTokenRounds. + // It MUST carry the SSPI/Negotiate token via StorageService.ValidateClientCredential, never via + // HistoryService.ExchangeKey (the earlier regression). ExchangeKey is now legitimately called + // directly in OpenSession for the SEPARATE v8 event-connection key exchange (ECDH) — that is + // allowed, so the guardrail scopes the no-ExchangeKey rule to the token-loop closures only. + HashSet tokenLoopCalls = CollectCalledMethodNames( + "AVEVA.Historian.Client.Grpc.HistorianGrpcHandshake", nestedClosuresOnly: true); - Assert.Contains("ValidateClientCredential", calledMethods); - Assert.DoesNotContain("ExchangeKey", calledMethods); + Assert.Contains("ValidateClientCredential", tokenLoopCalls); + Assert.DoesNotContain("ExchangeKey", tokenLoopCalls); } - private static HashSet CollectCalledMethodNames(string typeFullName) + private static HashSet CollectCalledMethodNames(string typeFullName, bool nestedClosuresOnly = false) { Assembly sdk = typeof(HistorianClientOptions).Assembly; Type orchestrator = sdk.GetType(typeFullName, throwOnError: true)!; Module module = orchestrator.Module; - // The orchestrator type plus its compiler-generated nested types (lambda closures). - IEnumerable types = new[] { orchestrator } - .Concat(orchestrator.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic)); + // The compiler-generated nested types (lambda closures) — optionally with the orchestrator type + // itself. The token loop lives inside a closure; ExchangeKey (event key exchange) lives in the + // OpenSession body, so scoping to closures isolates the token-loop routing. + IEnumerable nested = orchestrator.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic); + IEnumerable types = nestedClosuresOnly ? nested : new[] { orchestrator }.Concat(nested); var names = new HashSet(StringComparer.Ordinal); foreach (Type t in types) From 3fd522fa109b8ddbaa4e00ac09d2f4747489f24e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 10:31:37 -0400 Subject: [PATCH 4/9] =?UTF-8?q?docs(grpc-events):=20Path=20B=20=E2=80=94?= =?UTF-8?q?=20ExchangeKey=20ECDH=20clears=202=20of=203=20layers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records that the pure-managed P-256 ExchangeKey works (cleared the v8 client-key check; error advanced to 132/171 AuthenticationFailed). The remaining layer is the 26-byte credential-token KDF, which requires recovering the native key derivation. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md index cb56466..a95e4a8 100644 --- a/docs/reverse-engineering/grpc-event-query-capture.md +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -229,3 +229,28 @@ size) — using .NET `ECDiffieHellman`, establish the client key, then reissue t Open question for Path B: whether merely *completing* the ECDH key agreement registers the client key (so the zeroed openParameters token still rides through), or whether the token must also be derived from the shared secret (full KDF/cipher RE). + +### Path B started 2026-06-23 — ExchangeKey ECDH works; cleared 2 of 3 layers + +Implemented `HistoryService.ExchangeKey` as a **pure-managed P-256 ECDH** key exchange +(`HistorianNativeHandshake.BuildExchangeKeyClientHello` / `DeriveExchangeKeySecret`, .NET +`ECDiffieHellman` over `nistP256`; wire format `"ECK1" + u32(32) + X(32) + Y(32)`) and wired it into +`HistorianGrpcHandshake.OpenSession(eventConnection: true)` ahead of the v8 `OpenConnection`, +on the same context-key handle. Live result against the server: the **`ExchangeKey` RPC succeeds** +(the server accepted our public key), and the v8 `OpenConnection` error **moved one layer deeper**: + +``` +Path A (no ExchangeKey): 132/34 "Failed to get client key" +Path B (ExchangeKey ECDH): 132/171 AuthenticationFailed "EstablishConnection — Authentication failed" +``` + +So the ECDH cleared the client-key check; the remaining blocker is **authentication**: the 26-byte +v8 credential token must be a *valid* value derived from the ECDH shared secret (not zeros). This is +the token KDF/cipher — the part that is not yet reverse-engineered and that would require analyzing +AVEVA's native ExchangeKey/credential crypto to recover the derivation (the .NET-shipped result stays +pure managed either way). The "Path B-lite" hypothesis (zeroed token rides through after key +agreement) is therefore disproven at the auth layer — 2 of 3 layers are cleared, the 3rd is the +credential-token derivation. ExchangeKey + the v8 serializer are committed and ready; the orchestrator +stays on v6 (set `eventConnection: true` to re-arm once the token KDF lands). The token-loop routing +guardrail (`HistorianGrpcHandshakeRoutingTests`) was scoped to the closure so the legitimate +ExchangeKey call is allowed while still pinning that the Negotiate token loop never routes there. From b2ac35b98e1a21f669f8ac0e3147fe3724538a1c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 11:11:21 -0400 Subject: [PATCH 5/9] =?UTF-8?q?docs(grpc-events):=20trace=20the=20Exchange?= =?UTF-8?q?Key=20token=20crypto=20=E2=80=94=20KDF=3DSHA256(secret);=20toke?= =?UTF-8?q?n=20construction=20localized?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frida-hooked Windows CNG (scripts/frida/aahclientmanaged-cng-exchangekey.js) during a real native ExchangeKey to recover the token derivation: - The ECDH + KDF are standard CNG driven by managed System.Security.Cryptography .ECDiffieHellmanCng: NCryptSecretAgreement (P-256) -> NCryptDeriveKey(KDF=HASH, SHA256, 32 bytes). So the derived key = SHA256(ECDH shared secret). - "ECK1" is the standard CNG BCRYPT_ECCPUBLIC_BLOB magic (P-256), confirming our BuildExchangeKeyClientHello wire format. - The 26-byte token (constant 0x8e marker) is a custom construction over the derived key: a 528-candidate offline cracker (HMAC/SHA/AES-GCM/CBC/CTR over the derived key x request slices x creds) found no match, and it matches none of the traced hash digests. It is built in aahClientManaged's C++/CLI code between the DeriveKeyMaterial call and the openParameters assembly. Next: ILSpy cannot decompile the mixed-mode assembly (crashes, exit 70); use dnlib (IL-level) to dump the method referencing DeriveKeyMaterial and read the post-derive token construction. 2 of 3 layers cleared (key exchange + client key); the 3rd (token) is localized, pending dnlib extraction. Orchestrator stays on v6. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 44 ++++- .../frida/aahclientmanaged-cng-exchangekey.js | 173 ++++++++++++++++++ 2 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 scripts/frida/aahclientmanaged-cng-exchangekey.js diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md index a95e4a8..837f07f 100644 --- a/docs/reverse-engineering/grpc-event-query-capture.md +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -245,12 +245,38 @@ Path B (ExchangeKey ECDH): 132/171 AuthenticationFailed "EstablishConnection ``` So the ECDH cleared the client-key check; the remaining blocker is **authentication**: the 26-byte -v8 credential token must be a *valid* value derived from the ECDH shared secret (not zeros). This is -the token KDF/cipher — the part that is not yet reverse-engineered and that would require analyzing -AVEVA's native ExchangeKey/credential crypto to recover the derivation (the .NET-shipped result stays -pure managed either way). The "Path B-lite" hypothesis (zeroed token rides through after key -agreement) is therefore disproven at the auth layer — 2 of 3 layers are cleared, the 3rd is the -credential-token derivation. ExchangeKey + the v8 serializer are committed and ready; the orchestrator -stays on v6 (set `eventConnection: true` to re-arm once the token KDF lands). The token-loop routing -guardrail (`HistorianGrpcHandshakeRoutingTests`) was scoped to the closure so the legitimate -ExchangeKey call is allowed while still pinning that the Negotiate token loop never routes there. +v8 credential token must be a *valid* value derived from the ECDH shared secret (not zeros). + +### Token crypto traced 2026-06-23 (Frida → Windows CNG) — KDF found, token construction still open + +Hooked Windows CNG (`bcrypt.dll`/`ncrypt.dll`) while the native harness ran a real ExchangeKey +(`scripts/frida/aahclientmanaged-cng-exchangekey.js` + `artifacts/.../cng-trace.py`). Findings: + +- **The ECDH + KDF are standard CNG, driven by managed `System.Security.Cryptography.ECDiffieHellmanCng`** + (backtrace top frame = `System.Core.ni.dll`; the caller is aahClientManaged's C++/CLI ``): + `NCryptSecretAgreement` (P-256) → `NCryptDeriveKey(KDF=HASH, HASH_ALGORITHM=SHA256, 32 bytes)`. So the + derived key = **SHA256(ECDH shared secret)** — exactly `ECDiffieHellmanCng{ KeyDerivationFunction=Hash, + HashAlgorithm=SHA256 }.DeriveKeyMaterial(...)`. Our managed `DeriveExchangeKeySecret` should switch to + this (SHA256 of the raw agreement) to match. +- **`"ECK1"` is NOT AVEVA-custom** — it is the standard Windows CNG `BCRYPT_ECCPUBLIC_BLOB` magic for + P-256 (`NCryptExportKey`/`ImportKey` emit exactly `ECK1 + len(32) + X(32) + Y(32)`), confirming our + `BuildExchangeKeyClientHello` wire format is correct. +- **The 26-byte token is a custom construction that is not yet reproduced.** Correlated one run's + derived key (`SHA256(secret)`) with that run's token (from the IL openParameters capture): a + 528-candidate offline cracker (HMAC/SHA/AES-GCM/CBC/CTR over the derived key × request slices × + creds) found **no match**, and the token matches **none** of the traced hash digests. The token + starts with a constant `0x8e` marker in both captured runs (so it is structured, not raw cipher + output). It is built in managed code between the `DeriveKeyMaterial` call and the openParameters + assembly. + +**Next step:** ILSpy cannot decompile the mixed-mode assembly (full-assembly and `` both crash, +exit 70). Use **dnlib** (IL-level, won't choke on the native parts) to dump the `` method that +references `ECDiffieHellmanCng.DeriveKeyMaterial` and read the post-derive token construction, then +implement it managed-side and re-test (non-destructive). + +**2 of 3 layers cleared** (key exchange + client key); the 3rd (token construction) is localized to a +specific managed method, pending dnlib extraction. ExchangeKey + the v8 serializer are committed; the +orchestrator stays on v6 (set `eventConnection: true` to re-arm once the token construction lands). The +token-loop routing guardrail (`HistorianGrpcHandshakeRoutingTests`) was scoped to the closure so the +legitimate ExchangeKey call is allowed while still pinning that the Negotiate token loop never routes +there. diff --git a/scripts/frida/aahclientmanaged-cng-exchangekey.js b/scripts/frida/aahclientmanaged-cng-exchangekey.js new file mode 100644 index 0000000..8acb6d6 --- /dev/null +++ b/scripts/frida/aahclientmanaged-cng-exchangekey.js @@ -0,0 +1,173 @@ +// Frida hook for the native ExchangeKey credential-token crypto (Windows CNG / bcrypt.dll). +// Traces the ECDH secret agreement, the KDF (with its parameter list), symmetric-key import, and +// encrypt/hash so the 26-byte v8 credential-token derivation can be reconstructed in managed code. +// Reverse-engineering aid only — observes the native client; nothing is shipped from here. +'use strict'; + +function resolve(modName, fnName) { + let m = null; + try { m = Process.getModuleByName(modName); } catch (e) { + try { m = Module.load(modName); } catch (e2) { return null; } + } + try { return m.findExportByName(fnName); } catch (e) { return null; } +} + +function dump(label, ptr, len) { + if (ptr.isNull() || len <= 0) { console.log(label + ' '); return; } + const n = Math.min(len, 256); + console.log(label + ' (' + len + ' bytes)\n' + hexdump(ptr, { length: n, header: false, ansi: false })); +} + +function hook(modName, fnName, onEnter, onLeave) { + const addr = resolve(modName, fnName); + if (!addr) { console.log('[skip] ' + modName + '!' + fnName + ' not found'); return; } + Interceptor.attach(addr, { onEnter: onEnter, onLeave: onLeave }); + console.log('[hooked] ' + modName + '!' + fnName); +} + +// BCryptOpenAlgorithmProvider(phAlgorithm, pszAlgId, pszImplementation, dwFlags) — names every algo used. +hook('bcrypt.dll', 'BCryptOpenAlgorithmProvider', function (a) { + console.log('[OpenAlgorithmProvider] algId=' + (a[1].isNull() ? '?' : a[1].readUtf16String())); +}); + +// BCryptSecretAgreement(hPrivKey, hPubKey, *phAgreedSecret, flags) +hook('bcrypt.dll', 'BCryptSecretAgreement', function (a) { + console.log('[SecretAgreement] hPriv=' + a[0] + ' hPub=' + a[1]); +}); + +// Decode a BCryptBufferDesc parameter list (used by BCryptDeriveKey) into (type -> bytes). +function dumpParamList(pParamList) { + if (pParamList.isNull()) { console.log(' paramList '); return; } + const cBuffers = pParamList.add(4).readU32(); // ULONG ulVersion; ULONG cBuffers; + const pBuffers = pParamList.add(8).readPointer(); // BCryptBuffer* pBuffers; + const names = { 0: 'HASH_ALGORITHM', 1: 'SECRET_PREPEND', 2: 'SECRET_APPEND', 3: 'HMAC_KEY', + 4: 'TLS_PRF_LABEL', 5: 'TLS_PRF_SEED', 6: 'SECRET_HANDLE', 8: 'SP80056A_CONCAT', + 0xD: 'LABEL', 0xE: 'CONTEXT', 0xF: 'SALT', 0x10: 'ITERATION_COUNT' }; + console.log(' paramList cBuffers=' + cBuffers); + for (let i = 0; i < cBuffers; i++) { + const b = pBuffers.add(i * 16); // { ULONG cbBuffer; ULONG BufferType; PVOID pvBuffer; } + const cb = b.readU32(); + const type = b.add(4).readU32(); + const pv = b.add(8).readPointer(); + const tn = names[type] || ('0x' + type.toString(16)); + if (type === 0 || type === 4 || type === 0xD) { // string-ish (hash alg name / label) + console.log(' [' + tn + '] ' + (pv.isNull() ? '?' : pv.readUtf16String())); + } else { + dump(' [' + tn + ']', pv, cb); + } + } +} + +// BCryptDeriveKey(hSecret, pwszKDF, *pParamList, pbDerivedKey, cbDerivedKey, *pcbResult, flags) +hook('bcrypt.dll', 'BCryptDeriveKey', function (a) { + this.kdf = a[1].isNull() ? '?' : a[1].readUtf16String(); + this.outKey = a[3]; this.pcb = a[5]; + console.log('[DeriveKey] KDF=' + this.kdf + ' cbDerivedKey=' + a[4].toInt32()); + dumpParamList(a[2]); +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + dump('[DeriveKey] derived', this.outKey, n); +}); + +hook('bcrypt.dll', 'BCryptDeriveKeyPBKDF2', function (a) { + console.log('[PBKDF2] cbPassword=' + a[2].toInt32() + ' cbSalt=' + a[4].toInt32() + ' iter=' + a[5]); + dump(' password', a[1], a[2].toInt32()); + dump(' salt', a[3], a[4].toInt32()); +}); + +// BCryptGenerateSymmetricKey(hAlg, *phKey, pbKeyObject, cbKeyObject, pbSecret, cbSecret, flags) — the actual key bytes. +hook('bcrypt.dll', 'BCryptGenerateSymmetricKey', function (a) { + dump('[GenerateSymmetricKey] keyBytes', a[4], a[5].toInt32()); +}); + +// BCryptEncrypt(hKey, pbIn, cbIn, *pPad, pbIV, cbIV, pbOut, cbOut, *pcbResult, flags) +hook('bcrypt.dll', 'BCryptEncrypt', function (a) { + this.out = a[6]; this.pcb = a[8]; + dump('[Encrypt] plaintext', a[1], a[2].toInt32()); + dump('[Encrypt] IV', a[4], a[5].toInt32()); +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + dump('[Encrypt] ciphertext', this.out, n); +}); + +// Hash path (in case the token is a keyed hash rather than a cipher). +hook('bcrypt.dll', 'BCryptHashData', function (a) { + dump('[HashData] input', a[1], a[2].toInt32()); +}); +hook('bcrypt.dll', 'BCryptFinishHash', function (a) { + this.out = a[1]; this.cb = a[2].toInt32(); +}, function () { + dump('[FinishHash] digest', this.out, this.cb); +}); + +// ---- NCrypt (CNG key-storage layer) — the likely home of the ECDH ExchangeKey + token crypto ---- + +// NCryptSecretAgreement(hPrivKey, hPubKey, *phAgreedSecret, dwFlags) +hook('ncrypt.dll', 'NCryptSecretAgreement', function (a) { + console.log('[NCryptSecretAgreement] hPriv=' + a[0] + ' hPub=' + a[1]); + console.log(' backtrace (addr -> module+offset):'); + Thread.backtrace(this.context, Backtracer.ACCURATE).slice(0, 14).forEach(function (addr) { + const m = Process.findModuleByAddress(addr); + if (m) { + console.log(' ' + addr + ' ' + m.name + '+0x' + addr.sub(m.base).toString(16)); + } else { + console.log(' ' + addr + ' '); + } + }); +}); + +// NCryptDeriveKey(hSharedSecret, pwszKDF, *pParameterList, pbDerivedKey, cbDerivedKey, *pcbResult, dwFlags) +hook('ncrypt.dll', 'NCryptDeriveKey', function (a) { + this.kdf = a[1].isNull() ? '?' : a[1].readUtf16String(); + this.outKey = a[3]; this.pcb = a[5]; + console.log('[NCryptDeriveKey] KDF=' + this.kdf + ' cbDerivedKey=' + a[4].toInt32()); + dumpParamList(a[2]); +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + dump('[NCryptDeriveKey] derived', this.outKey, n); +}); + +// NCryptEncrypt(hKey, pbInput, cbInput, *pPaddingInfo, pbOutput, cbOutput, *pcbResult, dwFlags) +hook('ncrypt.dll', 'NCryptEncrypt', function (a) { + this.out = a[4]; this.pcb = a[6]; + dump('[NCryptEncrypt] plaintext', a[1], a[2].toInt32()); +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + dump('[NCryptEncrypt] ciphertext', this.out, n); +}); + +// NCryptImportKey(hProvider, hImportKey, pszBlobType, *pParameterList, *phKey, pbData, cbData, dwFlags) +hook('ncrypt.dll', 'NCryptImportKey', function (a) { + console.log('[NCryptImportKey] blobType=' + (a[2].isNull() ? '?' : a[2].readUtf16String())); + dump(' blob', a[5], a[6].toInt32()); +}); + +// NCryptExportKey(hKey, hExportKey, pszBlobType, *pParameterList, pbOutput, cbOutput, *pcbResult, dwFlags) +hook('ncrypt.dll', 'NCryptExportKey', function (a) { + this.blobType = a[2].isNull() ? '?' : a[2].readUtf16String(); + this.out = a[4]; this.pcb = a[6]; +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + console.log('[NCryptExportKey] blobType=' + this.blobType); + dump(' blob', this.out, n); +}); + +hook('ncrypt.dll', 'NCryptOpenStorageProvider', function (a) { + console.log('[NCryptOpenStorageProvider] ' + (a[1].isNull() ? '?' : a[1].readUtf16String())); +}); + +// BCrypt EC key operations (in case the ECDH is bcrypt but uses import/export rather than DeriveKey). +hook('bcrypt.dll', 'BCryptImportKeyPair', function (a) { + console.log('[BCryptImportKeyPair] blobType=' + (a[2].isNull() ? '?' : a[2].readUtf16String()) + ' cb=' + a[5].toInt32()); + dump(' blob', a[4], a[5].toInt32()); +}); +hook('bcrypt.dll', 'BCryptExportKey', function (a) { + this.blobType = a[2].isNull() ? '?' : a[2].readUtf16String(); + this.out = a[3]; this.pcb = a[5]; +}, function () { + const n = this.pcb.isNull() ? 0 : this.pcb.readU32(); + console.log('[BCryptExportKey] blobType=' + this.blobType); + dump(' blob', this.out, n); +}); + +console.log('=== CNG ExchangeKey crypto hooks installed ==='); From c45f1a957bfe7b59333aeb503deeabcd33195764 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 11:21:55 -0400 Subject: [PATCH 6/9] =?UTF-8?q?docs(grpc-events):=20token=20scheme=20fully?= =?UTF-8?q?=20RE'd=20via=20dnlib=20=E2=80=94=20aahCryptV2=20(MD5-keyed=20R?= =?UTF-8?q?C4=20+=20prefix)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loaded dnlib in PowerShell (ILSpy crashes on the mixed-mode assembly) and scanned the IL to recover the entire v8 token construction: - ::CHistoryConnectionGrpc.GetClientKey drives the ECDH: ECDiffieHellmanCng {KeyDerivationFunction=Hash, HashAlgorithm=SHA256, KeySize=256} -> ExchangeKey -> CngKey.Import(serverPub, EccPublicBlob) -> DeriveKeyMaterial = SHA256(shared secret), the 32-byte client key. - aahClientCommon.CClientBase.ConfigureOpenConnection (the lone GetClientKey caller) builds the 26-byte token via HistorianCrypto.NRC4_V2.aahCryptV2 = a custom MD5-keyed RC4 stream cipher with a version prefix: * body/HashData = MD5 (verified by the round constants 0xd76aa478... + shifts 7/12/17/22) * prepare_key = RC4 KSA from a 16-byte MD5 key * enc_buffer = MD5 -> key, then rc4encrypt; enc prepends PrefixV2/InnerPrefixV2 (the constant 0x8e token marker) So token = prefix + RC4(plaintext, key=MD5(keyMaterial)), keyMaterial tied to the SHA256(ECDH secret) client key. 100% reproducible in pure managed code (RC4+MD5). Remaining (next cycle): read ConfigureOpenConnection's exact key/plaintext/prefix bytes, implement aahCryptV2 managed-side, set the v8 token, live-test. Frida CNG + dnlib are the RE path; nothing AVEVA is shipped. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md index 837f07f..95f021e 100644 --- a/docs/reverse-engineering/grpc-event-query-capture.md +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -269,10 +269,36 @@ Hooked Windows CNG (`bcrypt.dll`/`ncrypt.dll`) while the native harness ran a re output). It is built in managed code between the `DeriveKeyMaterial` call and the openParameters assembly. -**Next step:** ILSpy cannot decompile the mixed-mode assembly (full-assembly and `` both crash, -exit 70). Use **dnlib** (IL-level, won't choke on the native parts) to dump the `` method that -references `ECDiffieHellmanCng.DeriveKeyMaterial` and read the post-derive token construction, then -implement it managed-side and re-test (non-destructive). +**dnlib IL extraction 2026-06-23 — the token scheme is fully reverse-engineered.** ILSpy can't +decompile the mixed-mode assembly (crashes), but loading `dnlib` in PowerShell and scanning the IL +recovered the whole construction: + +- **`::CHistoryConnectionGrpc.GetClientKey`** is the ECDH driver: `new ECDiffieHellmanCng()` + → `KeyDerivationFunction = Hash`, `HashAlgorithm = SHA256`, `KeySize = 256` → + `GrpcHistoryClient.ExchangeKey(strHandle, ourPubKey.ToByteArray(), out serverPub, out err)` → + `CngKey.Import(serverPub, CngKeyBlobFormat.EccPublicBlob)` → **`DeriveKeyMaterial`** = the 32-byte + client key = **`SHA256(ECDH shared secret)`**. (So our managed side should derive the key the same + way — `ECDiffieHellman` raw agreement then SHA256, or equivalently `DeriveKeyFromHash(..., SHA256)`.) +- **The 26-byte token is built by `aahClientCommon.CClientBase.ConfigureOpenConnection`** (the lone + caller of `GetClientKey`) using the **`HistorianCrypto.NRC4_V2.aahCryptV2`** scheme — a custom + **MD5-keyed RC4 stream cipher with a version prefix**: + - `aahCryptV2.body`/`HashData` = **MD5** (verified: the IL loads MD5 round constants `0xd76aa478`… + and rotates 7/12/17/22). + - `aahCryptV2.prepare_key` = standard **RC4 KSA** seeding the 256-byte S-box from a **16-byte (MD5)** + key (`std.array`). + - `aahCryptV2.enc_buffer` = `MD5(...)` → key, then **`rc4encrypt`** the body; `enc` prepends a + scheme **prefix** (`NRC4_V2.PrefixV2` / `InnerPrefixV2`) — the constant `0x8e` token marker. + - `from_GUID` keys the cipher from a GUID string. + +So the token = `prefix + RC4(plaintext, key = MD5(keyMaterial))`, where the key material ties back to +the `SHA256(ECDH secret)` client key. **This is 100% reproducible in pure managed code** (RC4 + MD5 +are ~40 lines; nothing AVEVA ships). + +**Remaining to finish (next cycle):** read `ConfigureOpenConnection`'s exact wiring (which value is +MD5'd for the RC4 key, what plaintext is encrypted, the exact prefix bytes — a little more dnlib IL), +implement `aahCryptV2` (RC4+MD5+prefix) managed-side, set the v8 token = that, and live-test +(non-destructive). The offline correlation data (one run's derived key + token + openParameters) is +captured under `artifacts/.../` to validate the managed reproduction before going live. **2 of 3 layers cleared** (key exchange + client key); the 3rd (token construction) is localized to a specific managed method, pending dnlib extraction. ExchangeKey + the v8 serializer are committed; the From 6d0f5c4b8ff786b80a9ced1ecc0490431efa47de Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 11:46:00 -0400 Subject: [PATCH 7/9] =?UTF-8?q?feat(grpc-events):=20implement=20aahCryptV2?= =?UTF-8?q?=20token=20=E2=80=94=20v8=20ExchangeKey=20auth=20now=20passes?= =?UTF-8?q?=20live?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the reverse-engineered v8 credential token in pure managed code and wires the full event-connection auth chain. Live result: the v8 OpenConnection now AUTHENTICATES against the 2023 R2 server (past the 132/171 AuthenticationFailed wall) — the crypto is solved. - HistorianNativeHandshake.DeriveExchangeKeyClientKey: client key = SHA256(ECDH shared secret) via ECDiffieHellman.DeriveKeyFromHash(SHA256), matching the native ECDiffieHellmanCng{Hash,SHA256}.DeriveKeyMaterial. - BuildExchangeKeyCredentialToken + Rc4: token = RC4(password-UTF16LE, key=MD5(clientKey)). Reproduces a live-captured token EXACTLY (verified offline) — the native HistorianCrypto.NRC4_V2.aahCryptV2 scheme (MD5-keyed RC4). Pure managed; nothing AVEVA shipped. RC4 pinned by the standard test vector. - OpenSession(eventConnection:true): ExchangeKey -> derive client key -> token -> v8 OpenConnection with ConnectionType=Event + the token. Orchestrator re-armed. - HistorianAddTagsProtocol.SerializeCmEventEnsureTagsGrpc: the 86-byte native gRPC CM_EVENT EnsureTags (8-byte header + ...2f27 event-type GUID), replacing the 2020 WCF 83-byte CTagMetadata on the gRPC event registration. Goldens: RC4 standard vector + token construction. 326/326 offline. KNOWN REMAINING: the event query still returns zero rows (GetNext yields a 10-byte zero-row buffer). Auth + StartEventQuery succeed; the query-layer detail (vs the native row-returning capture) is the last step. Gated test still pins the no-row throw; opt-in diagnostic (HISTORIAN_GRPC_EVENT_DIAG) surfaces the journey. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../Grpc/HistorianGrpcEventOrchestrator.cs | 16 +++--- .../Grpc/HistorianGrpcHandshake.cs | 16 +++--- .../Wcf/HistorianAddTagsProtocol.cs | 34 ++++++++++++ .../Wcf/HistorianNativeHandshake.cs | 53 +++++++++++++++++-- .../HistorianGrpcIntegrationTests.cs | 34 ++++++++++++ .../WcfOpen2ProtocolTests.cs | 26 +++++++++ 6 files changed, 160 insertions(+), 19 deletions(-) diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index 4d3ca7b..a61e88c 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -138,13 +138,10 @@ internal sealed class HistorianGrpcEventOrchestrator private List RunEventChain(DateTime startUtc, DateTime endUtc, HistorianEventFilter? filter, CancellationToken cancellationToken) { using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); - // Event reads need an Event-type (v8) connection. Path B established the v8 ExchangeKey (ECDH) - // client key — which cleared the "Failed to get client key" check — but the v8 OpenConnection - // then fails at native 132/171 AuthenticationFailed: the 26-byte credential token must be derived - // from the ECDH shared secret (the token-KDF is not yet reverse-engineered). Staying on the v6 - // session until that derivation lands; the ExchangeKey + v8 serializer are ready (set - // eventConnection: true to re-arm). See docs/reverse-engineering/grpc-event-query-capture.md. - HistorianGrpcHandshake.Session session = HistorianGrpcHandshake.OpenSession(connection, _options, cancellationToken); + // Event reads need an Event-type (v8) connection. OpenSession(eventConnection: true) runs the + // full v8 path: HistoryService.ExchangeKey (P-256 ECDH) -> client key = SHA256(secret) -> v8 + // OpenConnection with ConnectionType=Event and the credential token RC4(password, MD5(clientKey)). + HistorianGrpcHandshake.Session session = HistorianGrpcHandshake.OpenSession(connection, _options, cancellationToken, eventConnection: true); RegisterCmEventTag(connection, session, cancellationToken); @@ -253,7 +250,10 @@ internal sealed class HistorianGrpcEventOrchestrator TryRun(() => statusClient.GetStatusInterfaceVersion(new GrpcStatus.GetStatusInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); TryRun(() => retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - byte[] payload = HistorianAddTagsProtocol.SerializeCmEventCTagMetadata(DateTime.UtcNow); + // gRPC CM_EVENT EnsureTags uses the 86-byte native format (8-byte header + the …2f27 event-type + // GUID), NOT the 2020 WCF CTagMetadata — required for the server to establish CM_EVENT so the + // event query returns rows. + byte[] payload = HistorianAddTagsProtocol.SerializeCmEventEnsureTagsGrpc(DateTime.UtcNow); TryRun(() => historyClient.EnsureTags( new GrpcHistory.EnsureTagsRequest { StrHandle = session.StringHandle, BtTagInfos = ByteString.CopyFrom(payload), ElementCount = 1 }, connection.Metadata, RegistrationDeadline(), cancellationToken)); diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs index 4d5539d..0cbe42c 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs @@ -82,9 +82,12 @@ internal static class HistorianGrpcHandshake cancellationToken); // Event reads require an Event-type connection (ConnectionType=Event), which only the native - // v8 OpenConnection format carries — the v6 buffer has no such field. The v8 OpenConnection also - // looks up its client key in the registry HistoryService.ExchangeKey (ECDH) populates (not the - // one ValidateClientCredential does), so establish that key first via a P-256 key exchange. + // v8 OpenConnection format carries — the v6 buffer has no such field. The v8 path authenticates + // via HistoryService.ExchangeKey (P-256 ECDH): the shared secret -> SHA256 = the client key, and + // the v8 credential token = RC4(password-UTF16LE, key=MD5(clientKey)) (the native HistorianCrypto + // aahCryptV2 scheme). The server shares the secret and RC4-decrypts the token to validate the + // password. See docs/reverse-engineering/grpc-event-query-capture.md. + byte[] eventToken = []; if (eventConnection) { using ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256); @@ -103,12 +106,13 @@ internal static class HistorianGrpcHandshake throw new InvalidOperationException( $"gRPC ExchangeKey failed (errorLen={xkErr.Length}, native={xkDecoded?.Type}/{xkDecoded?.Code}, ascii='{xkAscii}')."); } - // The raw ECDH shared secret (if the v8 credential token later needs derivation) is: - // HistorianNativeHandshake.DeriveExchangeKeySecret(ecdh, xk.BtOutput?.ToByteArray() ?? []); + + byte[] clientKey = HistorianNativeHandshake.DeriveExchangeKeyClientKey(ecdh, xk.BtOutput?.ToByteArray() ?? []); + eventToken = HistorianNativeHandshake.BuildExchangeKeyCredentialToken(clientKey, options.Password); } byte[] open2Request = eventConnection - ? HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request(contextKey, options.UserName) + ? HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request(contextKey, options.UserName, eventToken) : HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); GrpcHistory.OpenConnectionResponse open2 = historyClient.OpenConnection( diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianAddTagsProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianAddTagsProtocol.cs index 009bc42..42c726a 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianAddTagsProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianAddTagsProtocol.cs @@ -44,6 +44,40 @@ internal static class HistorianAddTagsProtocol /// public static readonly Guid CommonArchestraEventTypeId = new("5f59ae42-3bb6-4760-91a5-ab0be01f9f02"); + /// + /// The CM_EVENT event-type GUID used by the 2023 R2 gRPC EnsureTags (captured ending + /// …e0 1f 2f 27) — distinct from the 2020 WCF capture's + /// (…9f02). + /// + public static readonly Guid CommonArchestraEventTypeIdGrpc = new("5f59ae42-3bb6-4760-91a5-ab0be01f2f27"); + + /// + /// Builds the native 2023 R2 gRPC CM_EVENT EnsureTags.tagInfos buffer (86 bytes, + /// captured byte-for-byte). Differs from the 2020 WCF : + /// it is wrapped in an 8-byte EnsureTags header (4E 67 03 00 01 00 00 00), uses the + /// event-type GUID, and has no trailing bytes after it. + /// Used by the gRPC event registration so the server actually establishes CM_EVENT and the event + /// query returns rows. + /// + public static byte[] SerializeCmEventEnsureTagsGrpc(DateTime createdUtc) + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream, Encoding.Unicode, leaveOpen: true); + + writer.Write(new byte[] { 0x4E, 0x67, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00 }); // EnsureTags header (count 1) + writer.Write((byte)3); + writer.Write((ushort)0x0086); + writer.Write((byte)5); + writer.Write(CmEventTagId.ToByteArray()); + WriteCompressedHistorianString(writer, "CM_EVENT"); + WriteCompressedHistorianString(writer, "AnE Event"); + writer.Write(new byte[] { 0x02, 0x02, 0x01, 0x00, 0x00, 0x00, 0x01 }); + writer.Write(0u); + writer.Write(createdUtc.ToUniversalTime().ToFileTimeUtc()); + writer.Write(CommonArchestraEventTypeIdGrpc.ToByteArray()); + return stream.ToArray(); + } + public static byte[] SerializeCmEventCTagMetadata(DateTime createdUtc) { using MemoryStream stream = new(); diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs index a2c2e93..723b364 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs @@ -56,9 +56,11 @@ internal static class HistorianNativeHandshake /// /// Parses the server's ExchangeKey hello (same "ECK1" + u32 + X + Y shape) and - /// derives the raw ECDH shared secret against . + /// derives the 32-byte client key = SHA256(ECDH shared secret). This matches the native + /// client, which uses ECDiffieHellmanCng { KeyDerivationFunction=Hash, HashAlgorithm=SHA256 } + /// .DeriveKeyMaterial(...) — i.e. the hash KDF over the raw agreement. /// - public static byte[] DeriveExchangeKeySecret(ECDiffieHellman ecdh, byte[] serverHello) + public static byte[] DeriveExchangeKeyClientKey(ECDiffieHellman ecdh, byte[] serverHello) { const int headerLength = 8; // 4-byte magic + 4-byte coordinate length int needed = headerLength + (2 * ExchangeKeyCoordinateBytes); @@ -76,7 +78,48 @@ internal static class HistorianNativeHandshake Q = new ECPoint { X = x, Y = y }, }; using ECDiffieHellman serverKey = ECDiffieHellman.Create(serverParameters); - return ecdh.DeriveRawSecretAgreement(serverKey.PublicKey); + return ecdh.DeriveKeyFromHash(serverKey.PublicKey, HashAlgorithmName.SHA256); + } + + /// + /// Builds the v8 credential token: RC4(password-UTF16LE, key = MD5(clientKey)), where + /// is the result + /// (SHA256 of the ECDH secret). Reverse-engineered from the native HistorianCrypto.NRC4_V2 + /// .aahCryptV2 scheme (MD5-keyed RC4) and verified to reproduce a live-captured token exactly. + /// The server, sharing the ECDH secret, RC4-decrypts this to recover and validate the password. + /// Pure managed; nothing AVEVA is shipped. + /// + public static byte[] BuildExchangeKeyCredentialToken(byte[] clientKey, string password) + { + byte[] rc4Key = MD5.HashData(clientKey); + byte[] plaintext = System.Text.Encoding.Unicode.GetBytes(password ?? string.Empty); + return Rc4(rc4Key, plaintext); + } + + internal static byte[] Rc4(byte[] key, byte[] data) + { + int[] s = new int[256]; + for (int i = 0; i < 256; i++) + { + s[i] = i; + } + + for (int i = 0, j = 0; i < 256; i++) + { + j = (j + s[i] + key[i % key.Length]) & 0xFF; + (s[i], s[j]) = (s[j], s[i]); + } + + byte[] output = new byte[data.Length]; + for (int i = 0, j = 0, n = 0; n < data.Length; n++) + { + i = (i + 1) & 0xFF; + j = (j + s[i]) & 0xFF; + (s[i], s[j]) = (s[j], s[i]); + output[n] = (byte)(data[n] ^ s[(s[i] + s[j]) & 0xFF]); + } + + return output; } private static byte[] LeftPadCoordinate(byte[] coordinate) @@ -218,7 +261,7 @@ internal static class HistorianNativeHandshake /// the v6 path already relies on (it sends a zeroed credential block). See /// docs/reverse-engineering/grpc-event-query-capture.md. /// - public static byte[] BuildEventOpenConnectionVersion8Request(Guid contextKey, string userName) + public static byte[] BuildEventOpenConnectionVersion8Request(Guid contextKey, string userName, byte[] credentialToken) { Process current = Process.GetCurrentProcess(); string machineName = Environment.MachineName; @@ -244,7 +287,7 @@ internal static class HistorianNativeHandshake NativeConnectionTypeEvent, NativeConnectionFlagEvent, userName ?? string.Empty, - credentialToken: new byte[CredentialTokenSizeBytes]); + credentialToken ?? []); } /// diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs index 9132e04..63e0517 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs @@ -515,6 +515,40 @@ public sealed class HistorianGrpcIntegrationTests }); } + [Fact] + public async Task EventReadDiagnostic_OverGrpc_PrintsJourney() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST"); + if (string.IsNullOrWhiteSpace(host) || string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HISTORIAN_USER"))) + { + return; + } + if (Environment.GetEnvironmentVariable("HISTORIAN_GRPC_EVENT_DIAG") is null) + { + return; // opt-in diagnostic only + } + + var orch = new AVEVA.Historian.Client.Grpc.HistorianGrpcEventOrchestrator(BuildOptions(host)); + var events = new List(); + string outcome; + try + { + await foreach (HistorianEvent evt in orch.ReadEventsAsync(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow, null, CancellationToken.None)) + { + events.Add(evt); + if (events.Count >= 3) { break; } + } + outcome = $"OK events={events.Count}"; + } + catch (Exception ex) + { + outcome = $"{ex.GetType().Name}: {ex.Message}"; + } + + throw new Xunit.Sdk.XunitException( + $"[DIAG] outcome={outcome} | events={events.Count} | LastResultLen={orch.LastResultBufferLength} | LastErr='{orch.LastErrorBufferDescription}'"); + } + [Fact] public async Task GetConnectionStatusAsync_OverGrpc_ReportsConnected() { diff --git a/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs index 3afd730..941b8f1 100644 --- a/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/WcfOpen2ProtocolTests.cs @@ -5,6 +5,32 @@ namespace AVEVA.Historian.Client.Tests; public sealed class WcfOpen2ProtocolTests { + [Fact] + public void Rc4MatchesStandardTestVector() + { + // Standard RC4 test vector (key "Key", plaintext "Plaintext" -> BBF316E8D940AF0AD3). Pins the + // RC4 used by the v8 credential token = RC4(password-UTF16LE, key=MD5(SHA256(ECDH secret))). + byte[] actual = HistorianNativeHandshake.Rc4( + System.Text.Encoding.ASCII.GetBytes("Key"), + System.Text.Encoding.ASCII.GetBytes("Plaintext")); + + Assert.Equal(Convert.FromHexString("BBF316E8D940AF0AD3"), actual); + } + + [Fact] + public void CredentialToken_IsRc4OfPasswordKeyedByMd5OfClientKey() + { + // token = RC4(password-UTF16LE, key = MD5(clientKey)). Verify against an independently computed + // reference for a fixed clientKey + password. + byte[] clientKey = Enumerable.Range(0, 32).Select(i => (byte)i).ToArray(); + byte[] token = HistorianNativeHandshake.BuildExchangeKeyCredentialToken(clientKey, "Pw1!"); + + byte[] rc4Key = System.Security.Cryptography.MD5.HashData(clientKey); + byte[] expected = HistorianNativeHandshake.Rc4(rc4Key, System.Text.Encoding.Unicode.GetBytes("Pw1!")); + Assert.Equal(expected, token); + Assert.Equal("Pw1!".Length * 2, token.Length); // UTF-16LE length + } + [Fact] public void Version8EventSerializerReproducesCapturedNativeStructure() { From 32ae301050c921a4b1a4db61b27728660bae2de0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 12:12:35 -0400 Subject: [PATCH 8/9] feat(grpc-events): native-match event registration + skip ValidateClientCredential; diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Continues closing the event-row gap after the v8 ExchangeKey/RC4 auth breakthrough. - HistorianGrpcHandshake: the v8 EVENT path skips StorageService.ValidateClientCredential (the native event connection authenticates purely via ExchangeKey + the RC4 token; running the Negotiate loop establishes a different session scope). - HistorianGrpcEventOrchestrator.RegisterCmEventTag: simplified to the exact native gRPC event sequence (UpdateClientStatus -> RegisterTags -> EnsureTags -> GetHistorianInfo -> GetSystemParameter x7), dropping the 2020-WCF-era cross-service GetV probes and params-before-register that the gRPC event flow does not use. eventCount 5 -> 100. - Opt-in diagnostics (RegistrationDiag, LastResultBufferHex/LastErrorBufferHex; gated EventReadDiagnostic test) for the continued investigation. STATUS: auth + StartEventQuery + registration all succeed live (RTag/EnsT=True, valid query handle), but GetNext returns version-11 rowCount-0 while the native returns 50 for a BYTE-IDENTICAL request. Every observable wire element matches the native. The remaining unknown is the string/uint HANDLE field VALUES the native uses per event RPC — the instrument-grpc capture logged only byte[] params, not the handle fields. Next: extend the IL rewrite to log strHandle/uiHandle/queryRequestType, re-capture, and match. 326/326 offline; gated test still pins the no-row throw. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../Grpc/HistorianGrpcEventOrchestrator.cs | 87 ++++++++++--------- .../Grpc/HistorianGrpcHandshake.cs | 41 +++++---- .../HistorianGrpcIntegrationTests.cs | 5 +- 3 files changed, 73 insertions(+), 60 deletions(-) diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index a61e88c..756685b 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -58,6 +58,12 @@ internal sealed class HistorianGrpcEventOrchestrator /// Diagnostic: type+code description of the most recent error/terminal buffer. public string LastErrorBufferDescription { get; private set; } = string.Empty; + /// Diagnostic: hex of the most recent result buffer (first 48 bytes). + public string LastResultBufferHex { get; private set; } = string.Empty; + + /// Diagnostic: hex of the most recent GetNext error buffer. + public string LastErrorBufferHex { get; private set; } = string.Empty; + public async IAsyncEnumerable ReadEventsAsync( DateTime startUtc, DateTime endUtc, @@ -206,59 +212,55 @@ internal sealed class HistorianGrpcEventOrchestrator { var historyClient = new GrpcHistory.HistoryService.HistoryServiceClient(connection.Channel); var statusClient = new GrpcStatus.StatusService.StatusServiceClient(connection.Channel); - var retrievalClient = new GrpcRetrieval.RetrievalService.RetrievalServiceClient(connection.Channel); - var transactionClient = new GrpcTransaction.TransactionService.TransactionServiceClient(connection.Channel); - - // Discovery dance the native event flow runs between Open2 and EnsT2. All bounded by the - // short RegistrationDeadline (several stall server-side on the remote box). - TryRun(() => statusClient.GetStatusInterfaceVersion(new GrpcStatus.GetStatusInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - TryRun(() => statusClient.GetStatusInterfaceVersion(new GrpcStatus.GetStatusInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - - byte[] historianVersionRequest = HistorianEventRegistrationProtocol.BuildGetHistorianInfoRequest("HistorianVersion"); - TryRun(() => statusClient.GetHistorianInfo( - new GrpcStatus.GetHistorianInfoRequest { StrHandle = session.StringHandle, BtRequest = ByteString.CopyFrom(historianVersionRequest) }, - connection.Metadata, RegistrationDeadline(), cancellationToken)); - TryRun(() => statusClient.GetHistorianInfo( - new GrpcStatus.GetHistorianInfoRequest { StrHandle = session.StringHandle, BtRequest = ByteString.CopyFrom(historianVersionRequest) }, - connection.Metadata, RegistrationDeadline(), cancellationToken)); + // Native 2023 R2 gRPC event-connection registration sequence (captured order): + // UpdateClientStatus -> RegisterTags(CM_EVENT) -> EnsureTags(CM_EVENT) -> GetHistorianInfo + // -> GetSystemParameter x7. (StartEventQuery follows in RunEventQuery.) The 2020-WCF-era extra + // probes (cross-service GetV, params-before-register) are NOT in the gRPC event flow. byte[] clientStatus = HistorianEventRegistrationProtocol.BuildUpdateClientStatusBlob(); TryRun(() => historyClient.UpdateClientStatus( new GrpcHistory.UpdateClientStatusRequest { StrHandle = session.StringHandle, BtClientStatus = ByteString.CopyFrom(clientStatus) }, connection.Metadata, RegistrationDeadline(), cancellationToken)); - // Records 11-16: 6 system-parameter queries before RTag2. - foreach (string parameterName in HistorianEventRegistrationProtocol.StatusParametersBeforeRegister) + byte[] registerBuffer = HistorianEventRegistrationProtocol.BuildRegisterCmEventInputBuffer(); + try + { + GrpcHistory.RegisterTagsResponse rt = historyClient.RegisterTags( + new GrpcHistory.RegisterTagsRequest { StrHandle = session.StringHandle, BtTagInfos = ByteString.CopyFrom(registerBuffer) }, + connection.Metadata, RegistrationDeadline(), cancellationToken); + RegistrationDiag += $"RTag={rt.Status?.BSuccess} e={Convert.ToHexString(rt.Status?.BtError?.ToByteArray() ?? [])}; "; + } + catch (Exception ex) { RegistrationDiag += $"RTag=EX:{ex.GetType().Name}; "; } + + // gRPC CM_EVENT EnsureTags uses the 86-byte native format (8-byte header + the …2f27 event-type + // GUID), NOT the 2020 WCF CTagMetadata. + byte[] payload = HistorianAddTagsProtocol.SerializeCmEventEnsureTagsGrpc(DateTime.UtcNow); + try + { + GrpcHistory.EnsureTagsResponse et = historyClient.EnsureTags( + new GrpcHistory.EnsureTagsRequest { StrHandle = session.StringHandle, BtTagInfos = ByteString.CopyFrom(payload), ElementCount = 1 }, + connection.Metadata, RegistrationDeadline(), cancellationToken); + RegistrationDiag += $"EnsT={et.Status?.BSuccess} e={Convert.ToHexString(et.Status?.BtError?.ToByteArray() ?? [])} out={Convert.ToHexString(et.BtTagStatus?.ToByteArray() ?? [])}; "; + } + catch (Exception ex) { RegistrationDiag += $"EnsT=EX:{ex.GetType().Name}; "; } + + byte[] historianVersionRequest = HistorianEventRegistrationProtocol.BuildGetHistorianInfoRequest("HistorianVersion"); + TryRun(() => statusClient.GetHistorianInfo( + new GrpcStatus.GetHistorianInfoRequest { StrHandle = session.StringHandle, BtRequest = ByteString.CopyFrom(historianVersionRequest) }, + connection.Metadata, RegistrationDeadline(), cancellationToken)); + + string[] eventParams = ["AllowOriginals", "HistorianPartner", "HistorianVersion", "MaxCyclicStorageTimeout", "RealTimeWindow", "FutureTimeThreshold", "AllowRenameTags"]; + foreach (string parameterName in eventParams) { TryRun(() => statusClient.GetSystemParameter( new GrpcStatus.GetSystemParameterRequest { UiHandle = session.ClientHandle, StrParameterName = parameterName }, connection.Metadata, RegistrationDeadline(), cancellationToken)); } - - byte[] registerBuffer = HistorianEventRegistrationProtocol.BuildRegisterCmEventInputBuffer(); - TryRun(() => historyClient.RegisterTags( - new GrpcHistory.RegisterTagsRequest { StrHandle = session.StringHandle, BtTagInfos = ByteString.CopyFrom(registerBuffer) }, - connection.Metadata, RegistrationDeadline(), cancellationToken)); - - // Record 18: one more system-parameter query after RTag2 before EnsT2. - TryRun(() => statusClient.GetSystemParameter( - new GrpcStatus.GetSystemParameterRequest { UiHandle = session.ClientHandle, StrParameterName = "AllowRenameTags" }, - connection.Metadata, RegistrationDeadline(), cancellationToken)); - - // Records 19-21: cross-service version probes between RTag2 and EnsT2 (session-table registration). - TryRun(() => transactionClient.GetTransactionInterfaceVersion(new GrpcTransaction.GetTransactionInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - TryRun(() => statusClient.GetStatusInterfaceVersion(new GrpcStatus.GetStatusInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - TryRun(() => retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), connection.Metadata, RegistrationDeadline(), cancellationToken)); - - // gRPC CM_EVENT EnsureTags uses the 86-byte native format (8-byte header + the …2f27 event-type - // GUID), NOT the 2020 WCF CTagMetadata — required for the server to establish CM_EVENT so the - // event query returns rows. - byte[] payload = HistorianAddTagsProtocol.SerializeCmEventEnsureTagsGrpc(DateTime.UtcNow); - TryRun(() => historyClient.EnsureTags( - new GrpcHistory.EnsureTagsRequest { StrHandle = session.StringHandle, BtTagInfos = ByteString.CopyFrom(payload), ElementCount = 1 }, - connection.Metadata, RegistrationDeadline(), cancellationToken)); } + /// Diagnostic: outcomes of the key CM_EVENT registration RPCs. + public string RegistrationDiag { get; private set; } = string.Empty; + private List RunEventQuery( HistorianGrpcConnection connection, HistorianGrpcHandshake.Session session, @@ -280,7 +282,7 @@ internal sealed class HistorianGrpcEventOrchestrator IReadOnlyList attempts = HistorianEventQueryProtocol.CreateStartEventQueryAttempts( startUtc.ToUniversalTime(), endUtc.ToUniversalTime(), - eventCount: 5, + eventCount: 100, filter, version: 6); byte[] requestBuffer = attempts[0].RequestBuffer; @@ -304,6 +306,7 @@ internal sealed class HistorianGrpcEventOrchestrator } uint queryHandle = startResponse.UiQueryHandle; + RegistrationDiag += $"QH={queryHandle} clientH={session.ClientHandle} SEQresp={Convert.ToHexString(startResponse.BtResonse?.ToByteArray() ?? [])}; "; try { List events = []; @@ -339,6 +342,8 @@ internal sealed class HistorianGrpcEventOrchestrator LastResultBufferLength = resultBuffer.Length; LastErrorBufferDescription = HistorianEventRegistrationProtocol.DescribeNativeError(errorBuffer); + LastResultBufferHex = Convert.ToHexString(resultBuffer.Length <= 48 ? resultBuffer : resultBuffer[..48]); + LastErrorBufferHex = Convert.ToHexString(errorBuffer); // Any 5-byte type=4 error is a soft terminal (code 30 NoMoreData is canonical; code // 85 / 0x55 is the missing-registration signal seen on early runs). Mirror the WCF diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs index 0cbe42c..5307ab7 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs @@ -63,23 +63,30 @@ internal static class HistorianGrpcHandshake new GrpcHistory.GetInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); HistorianServerVersionGate.Validate(HistorianServiceInterface.History, historyVersion.UiVersion, options); - var storageClient = new GrpcStorage.StorageService.StorageServiceClient(connection.Channel); - HistorianNativeHandshake.RunTokenRounds( - (handle, wrapped, _) => - { - GrpcStorage.ValidateClientCredentialResponse response = storageClient.ValidateClientCredential( - new GrpcStorage.ValidateClientCredentialRequest { Handle = handle, InBuff = ByteString.CopyFrom(wrapped) }, - connection.Metadata, - Deadline(), - cancellationToken); - byte[] serverOutput = response.OutBuff?.ToByteArray() ?? []; - byte[] error = response.Status?.BtError?.ToByteArray() ?? []; - bool success = response.Status?.BSuccess ?? false; - return new HistorianNativeHandshake.TokenExchangeResult(success, serverOutput, error); - }, - contextKey, - options, - cancellationToken); + // The v6 (read/write) path authenticates via StorageService.ValidateClientCredential (Negotiate). + // The v8 EVENT path authenticates entirely via ExchangeKey (ECDH) + the RC4 credential token — + // the native client does NOT run ValidateClientCredential for an event connection, and doing so + // establishes a different session scope under which the event query returns zero rows. So skip it. + if (!eventConnection) + { + var storageClient = new GrpcStorage.StorageService.StorageServiceClient(connection.Channel); + HistorianNativeHandshake.RunTokenRounds( + (handle, wrapped, _) => + { + GrpcStorage.ValidateClientCredentialResponse response = storageClient.ValidateClientCredential( + new GrpcStorage.ValidateClientCredentialRequest { Handle = handle, InBuff = ByteString.CopyFrom(wrapped) }, + connection.Metadata, + Deadline(), + cancellationToken); + byte[] serverOutput = response.OutBuff?.ToByteArray() ?? []; + byte[] error = response.Status?.BtError?.ToByteArray() ?? []; + bool success = response.Status?.BSuccess ?? false; + return new HistorianNativeHandshake.TokenExchangeResult(success, serverOutput, error); + }, + contextKey, + options, + cancellationToken); + } // Event reads require an Event-type connection (ConnectionType=Event), which only the native // v8 OpenConnection format carries — the v6 buffer has no such field. The v8 path authenticates diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs index 63e0517..549cea1 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs @@ -533,7 +533,7 @@ public sealed class HistorianGrpcIntegrationTests string outcome; try { - await foreach (HistorianEvent evt in orch.ReadEventsAsync(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow, null, CancellationToken.None)) + await foreach (HistorianEvent evt in orch.ReadEventsAsync(DateTime.UtcNow.AddDays(-90), DateTime.UtcNow, null, CancellationToken.None)) { events.Add(evt); if (events.Count >= 3) { break; } @@ -546,7 +546,8 @@ public sealed class HistorianGrpcIntegrationTests } throw new Xunit.Sdk.XunitException( - $"[DIAG] outcome={outcome} | events={events.Count} | LastResultLen={orch.LastResultBufferLength} | LastErr='{orch.LastErrorBufferDescription}'"); + $"[DIAG] outcome={outcome} | events={events.Count} | LastResultLen={orch.LastResultBufferLength} " + + $"| ResultHex={orch.LastResultBufferHex} | ErrHex={orch.LastErrorBufferHex} | Reg=[{orch.RegistrationDiag}]"); } [Fact] From 0921e21bdb736f36abf8dd00d3cfa5ee180cc4ce Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Tue, 23 Jun 2026 12:31:33 -0400 Subject: [PATCH 9/9] =?UTF-8?q?feat(grpc-events):=20handle-capture=20cycle?= =?UTF-8?q?=20=E2=80=94=20event-row=20gap=20proven=20NOT=20a=20client=20pa?= =?UTF-8?q?yload=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the instrument-grpc rewrite to log string (strHandle) + uint (uiHandle / queryRequestType) params, not just byte[], and captures our SDK's live v8 openParameters for a byte-diff against the native. Result of the exhaustive comparison (all live-confirmed via the opt-in EventReadDiagnostic test): - StartEventQuery request: byte-identical to the native (v6 layout) - v8 OpenConnection openParameters: byte-identical to the native (302B) once ClientNodeName matches — every control byte/ConnectionType/token/ShardId - handle usage identical: ExchangeKey->contextKey, registration->storage GUID (strHandle), query->client uint (uiHandle); handles valid (RTag/EnsT=True) - queryRequestType=3, registration order, gzip metadata header — all match - window has events (native returns 50 now); eventCount not it Every observable client-side byte matches the native, yet the server scopes 0 events to our connection. The event RPCs succeed over our transport and return a valid EMPTY result (not a transport error), so this is a connection/server-level difference (session affinity tied to the native Grpc.Core HTTP/2 connection or a connection identity used to scope events) — invisible to and unfixable by client payload matching. Needs server-side insight, not more wire RE. Added opt-in diagnostics (RegistrationDiag, LastResultBufferHex, LastEventOpenRequestHex). 326/326 offline; gated test still pins the no-row throw. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01B6mcaT2PjRFKcogzp9UkfC --- .../grpc-event-query-capture.md | 33 +++++++++++++++++++ .../Grpc/HistorianGrpcEventOrchestrator.cs | 2 +- .../Grpc/HistorianGrpcHandshake.cs | 4 +++ .../HistorianGrpcIntegrationTests.cs | 6 ++-- .../Program.cs | 32 ++++++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/docs/reverse-engineering/grpc-event-query-capture.md b/docs/reverse-engineering/grpc-event-query-capture.md index 95f021e..131cf7b 100644 --- a/docs/reverse-engineering/grpc-event-query-capture.md +++ b/docs/reverse-engineering/grpc-event-query-capture.md @@ -300,6 +300,39 @@ implement `aahCryptV2` (RC4+MD5+prefix) managed-side, set the v8 token = that, a (non-destructive). The offline correlation data (one run's derived key + token + openParameters) is captured under `artifacts/.../` to validate the managed reproduction before going live. +### Token implemented + auth WORKS live (2026-06-23); row retrieval still 0 — proven NOT a payload issue + +`token = RC4(password-UTF16LE, key = MD5(SHA256(ECDH secret)))` was implemented in pure managed C# +(`HistorianNativeHandshake.BuildExchangeKeyCredentialToken` + `Rc4`; client key via +`DeriveKeyFromHash(SHA256)`), golden-tested (RC4 standard vector + token construction), and +**live-verified**: the v8 `OpenConnection` now **authenticates** against the 2023 R2 server (past the +`132/171 AuthenticationFailed` wall). Auth is solved. + +The event **query** still returns `version-11 rowCount-0` while the native returns 50 for an +**identical** request. Exhaustively ruled out as the cause (all confirmed live, opt-in +`EventReadDiagnostic` test + the IL rewrite extended to log string/uint handle fields): + +- `StartEventQuery` request: **byte-identical** to the native (v6 layout) +- v8 `OpenConnection` `openParameters`: **byte-identical** to the native (302 bytes) once ClientNodeName + is matched — every control byte, ConnectionType, token framing, ShardId, etc. +- Handle usage: identical — `ExchangeKey`→contextKey, registration→storage-session GUID (`strHandle`), + query→client uint (`uiHandle`); our parsed handles are valid (registration `RTag/EnsT=True`, valid + `queryHandle`) +- `queryRequestType = 3`, registration sequence/order, gzip metadata header — all match +- window (events exist; native returns 50 *now*), eventCount — not it + +So **every observable client-side byte matches the native**, yet the server scopes 0 events to our +connection. The event RPCs succeed over our transport and return a valid *empty* result (not a +transport error), so it is **not a payload or transport-incompatibility issue** — it is a +connection/server-level difference (e.g. session affinity tied to the native `Grpc.Core` HTTP/2 +connection or a connection-identity the server uses to scope events) that is **invisible to, and +unfixable by, client payload matching.** Closing it needs server-side insight or a different angle +(e.g. compare the full HTTP/2 connection setup / TLS identity), not more wire-payload RE. + +**Shipped this effort:** the complete ExchangeKey crypto (ECDH + SHA256 + MD5-keyed RC4 token) — the +hard wall — pure managed, golden-tested, auth live-verified. Orchestrator stays on the no-row throw; +gated test unchanged. + **2 of 3 layers cleared** (key exchange + client key); the 3rd (token construction) is localized to a specific managed method, pending dnlib extraction. ExchangeKey + the v8 serializer are committed; the orchestrator stays on v6 (set `eventConnection: true` to re-arm once the token construction lands). The diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs index 756685b..d9879cf 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcEventOrchestrator.cs @@ -306,7 +306,7 @@ internal sealed class HistorianGrpcEventOrchestrator } uint queryHandle = startResponse.UiQueryHandle; - RegistrationDiag += $"QH={queryHandle} clientH={session.ClientHandle} SEQresp={Convert.ToHexString(startResponse.BtResonse?.ToByteArray() ?? [])}; "; + RegistrationDiag += $"QH={queryHandle} clientH={session.ClientHandle} strH={session.StringHandle}; "; try { List events = []; diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs index 5307ab7..3e14d4e 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHandshake.cs @@ -21,6 +21,9 @@ namespace AVEVA.Historian.Client.Grpc; /// internal static class HistorianGrpcHandshake { + /// Diagnostic: hex of the most recent v8 event-connection OpenConnection request. + internal static string LastEventOpenRequestHex { get; private set; } = string.Empty; + /// /// The handles produced by a successful OpenConnection. is the /// transient uint session token used by StartQuery/GetSystemParameter and the other @@ -121,6 +124,7 @@ internal static class HistorianGrpcHandshake byte[] open2Request = eventConnection ? HistorianNativeHandshake.BuildEventOpenConnectionVersion8Request(contextKey, options.UserName, eventToken) : HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); + if (eventConnection) { LastEventOpenRequestHex = Convert.ToHexString(open2Request); } GrpcHistory.OpenConnectionResponse open2 = historyClient.OpenConnection( new GrpcHistory.OpenConnectionRequest { BtConnectionRequest = ByteString.CopyFrom(open2Request) }, diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs index 549cea1..9ff5739 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs @@ -547,7 +547,8 @@ public sealed class HistorianGrpcIntegrationTests throw new Xunit.Sdk.XunitException( $"[DIAG] outcome={outcome} | events={events.Count} | LastResultLen={orch.LastResultBufferLength} " + - $"| ResultHex={orch.LastResultBufferHex} | ErrHex={orch.LastErrorBufferHex} | Reg=[{orch.RegistrationDiag}]"); + $"| ResultHex={orch.LastResultBufferHex} | Reg=[{orch.RegistrationDiag}] " + + $"| v8open={AVEVA.Historian.Client.Grpc.HistorianGrpcHandshake.LastEventOpenRequestHex}"); } [Fact] @@ -677,7 +678,8 @@ public sealed class HistorianGrpcIntegrationTests IntegratedSecurity = !explicitCreds, UserName = user ?? string.Empty, Password = password ?? string.Empty, - RequestTimeout = timeout + RequestTimeout = timeout, + Compression = true // the stock client always advertises grpc gzip request encoding }; } } diff --git a/tools/AVEVA.Historian.ReverseEngineering/Program.cs b/tools/AVEVA.Historian.ReverseEngineering/Program.cs index 62bbdda..3ec8de3 100644 --- a/tools/AVEVA.Historian.ReverseEngineering/Program.cs +++ b/tools/AVEVA.Historian.ReverseEngineering/Program.cs @@ -1387,10 +1387,14 @@ static int InstrumentGrpcNonStream(string[] args) ModuleDefMD module = ModuleDefMD.Load(sourcePath); MemberRefUser logByteArray = CreateLogByteArrayRef(module); + MemberRefUser logString = CreateLogStringRef(module); + MemberRefUser logUInt32 = CreateLogUInt32Ref(module); // Cast a wide net: instrument EVERY byte[]-input method on every Grpc*Client type, so whichever // path the native non-streamed write actually drives (History/Transaction RegisterTags + // AddNonStreamValues, or a Storage-service route) is captured. Phase = "..". + // Also captures string (strHandle) and uint (uiHandle / queryRequestType) inputs so the event-read + // handle fields are visible, not just byte[] params. var instrumented = new List(); foreach (TypeDef type in module.GetTypes() .Where(t => t.Name.String.StartsWith("Grpc", StringComparison.Ordinal) && t.Name.String.EndsWith("Client", StringComparison.Ordinal))) @@ -1431,6 +1435,34 @@ static int InstrumentGrpcNonStream(string[] args) }); } + // ENTRY: log string (strHandle) and uint (uiHandle / queryRequestType / count) inputs. + foreach (dnlib.DotNet.Parameter scalarParam in method.Parameters + .Where(p => !p.IsHiddenThisParameter && (p.Type.FullName == "System.String" || p.Type.FullName == "System.UInt32")) + .ToArray()) + { + bool isString = scalarParam.Type.FullName == "System.String"; + string phase = $"{type.Name}.{method.Name}.{scalarParam.Name}.{(isString ? "str" : "u32")}"; + Instruction[] injected = + [ + Instruction.Create(OpCodes.Ldstr, phase), + Instruction.Create(OpCodes.Ldarg, scalarParam), + Instruction.Create(OpCodes.Call, isString ? logString : logUInt32), + ]; + foreach (Instruction instruction in injected.Reverse()) + { + method.Body.Instructions.Insert(0, instruction); + } + method.Body.MaxStack = (ushort)Math.Max((int)method.Body.MaxStack, 8); + instrumented.Add(new + { + Type = type.Name.String, + Method = method.Name.String, + Phase = phase, + Direction = "in", + Token = "0x" + method.MDToken.Raw.ToString("X8"), + }); + } + // EXIT: log out/ref byte[] responses ("System.Byte[]&") before each ret. ldarg loads the // managed pointer; ldind.ref dereferences it to the byte[]. (RPC wrappers set the out // param right before a single ret, so branch-to-ret skew is not a concern here.)