diff --git a/.gitignore b/.gitignore index 463f374..680e2bb 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ Thumbs.db # Test droppings *.coverage coverage.cobertura.xml + +# Live 2023 R2 server credentials — never commit +wonder-sql-vd03.txt diff --git a/CLAUDE.md b/CLAUDE.md index 92f1e90..e4331e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,7 +71,7 @@ Three layered subsystems, intentionally decoupled so protocol parsing can be uni - **`Wcf/`** — managed WCF/MDAS layer. The Historian uses Net.TCP on port `32568` with a custom `application/x-mdas` content type wrapping a binary SOAP 1.2 / WS-Addressing 1.0 envelope. `MdasMessageEncoder` + `MdasMessageEncodingBindingElement` implement that wrapper. `HistorianWcfBindingFactory` produces three flavors: plain MDAS, MDAS+Windows transport (used for `/Hist-Integrated`), and MDAS+certificate (used for `/HistCert`). Service paths live in `HistorianWcfServiceNames`. WCF data contracts (`Wcf/Contracts/`) are reproduced from server-side static analysis and are versioned per native interface (e.g., `IRetrievalServiceContract2..4`). - **`Protocol/`** — binary frame layer (`HistorianFrameReader`/`Writer`, `HistorianBinaryPrimitives`, `HistorianMessageType`). `Historian2020ProtocolDialect` is the version-anchored bridge between `HistorianClient` and the frame layer; methods without sufficient evidence throw `ProtocolEvidenceMissingException` rather than guessing wire bytes. - **`Transport/`** — pluggable `IHistorianTransport` (default: TCP). Tests inject a fake transport. -- **`Grpc/`** — 2023 R2 gRPC transport (`HistorianTransport.RemoteGrpc`). The recovered protobuf contract lives in `Grpc/Protos/*.proto` and is compiled to client stubs at build time by `Grpc.Tools`. `HistorianGrpcChannelFactory` builds a gRPC-Web/HTTP-1.1 channel (default port `32565`, optional TLS, gzip) matching the stock 2023 R2 client. `HistorianGrpcReadOrchestrator` mirrors `HistorianWcfReadOrchestrator` but over gRPC: it reuses the exact native serializers/parsers — the same Open2 buffer, SSPI/NTLM tokens, and `DataQueryRequest`/result buffers travel inside protobuf `bytes` fields. The 2020→gRPC op map: `Hist.ValCl`→`HistoryService.ExchangeKey`, `Hist.Open2`→`HistoryService.OpenConnection`, `Retr.StartQuery2`→`RetrievalService.StartQuery`, `Retr.GetNextQueryResultBuffer2`→`RetrievalService.GetNextQueryResultBuffer`. The transport-agnostic handshake (Open2 request builder + SSPI token loop + response decode) is shared via `Wcf/HistorianNativeHandshake`. **Not yet live-verified against a 2023 R2 server** — the auth handshake op (`ExchangeKey`) is the first thing to revisit if a live server rejects it; the byte payloads are the proven 2020 protocol. Gated live test: set `HISTORIAN_GRPC_HOST` (+ `HISTORIAN_TEST_TAG`, optional `HISTORIAN_GRPC_PORT`/`HISTORIAN_GRPC_TLS`/`HISTORIAN_GRPC_DNSID`). +- **`Grpc/`** — 2023 R2 gRPC transport (`HistorianTransport.RemoteGrpc`). The recovered protobuf contract lives in `Grpc/Protos/*.proto` and is compiled to client stubs at build time by `Grpc.Tools`. `HistorianGrpcChannelFactory` builds a gRPC-Web/HTTP-1.1 channel (default port `32565`, optional TLS, gzip) matching the stock 2023 R2 client. `HistorianGrpcReadOrchestrator` mirrors `HistorianWcfReadOrchestrator` but over gRPC: it reuses the exact native serializers/parsers — the same Open2 buffer, SSPI/NTLM tokens, and `DataQueryRequest`/result buffers travel inside protobuf `bytes` fields. The 2020→gRPC op map: `Hist.ValCl`→`StorageService.ValidateClientCredential` (the SSPI/Negotiate token loop), `Hist.Open2`→`HistoryService.OpenConnection`, `Retr.StartQuery2`→`RetrievalService.StartQuery`, `Retr.GetNextQueryResultBuffer2`→`RetrievalService.GetNextQueryResultBuffer`. The transport-agnostic handshake (Open2 request builder + SSPI token loop + response decode) is shared via `Wcf/HistorianNativeHandshake`. **Live-verified 2026-06-21 against a real 2023 R2 server** (interface versions History=12, Retrieval=4, Storage=4): the full read chain returns rows. NOTE: `HistoryService.ExchangeKey` is a SEPARATE key-exchange/cert-path op, NOT the Negotiate loop — an earlier revision wrongly routed the token loop there and it was rejected at round 0 regardless of credentials; the loop belongs on `StorageService.ValidateClientCredential` (which kept the 2020 inBuff/outBuff token framing). The byte payloads are the proven 2020 protocol and transfer unchanged; only the History interface integer differs (12 vs 11) and is buffer-compatible, so `VerifyServerInterfaceVersion=false` is currently required against a v12 server (the gate still pins History=11). Gated live test: set `HISTORIAN_GRPC_HOST` (+ `HISTORIAN_TEST_TAG`, optional `HISTORIAN_GRPC_PORT`/`HISTORIAN_GRPC_TLS`/`HISTORIAN_GRPC_DNSID`); reach the live 2023 R2 box via [[reference_2023r2_live_server_access]]. - **`Models/`** — public DTOs and enums (`HistorianSample`, `RetrievalMode`, etc.). `HistorianDataValue` represents the discriminated value type. `InternalsVisibleTo` exposes internals to the test assembly and the reverse-engineering tool. diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs index 72d2d5d..979ec9a 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs @@ -5,6 +5,7 @@ using AVEVA.Historian.Client.Models; using AVEVA.Historian.Client.Wcf; using GrpcHistory = ArchestrA.Grpc.Contract.History; using GrpcRetrieval = ArchestrA.Grpc.Contract.Retrieval; +using GrpcStorage = ArchestrA.Grpc.Contract.Storage; namespace AVEVA.Historian.Client.Grpc; @@ -16,17 +17,23 @@ namespace AVEVA.Historian.Client.Grpc; /// /// Operation mapping (2020 WCF → 2023 R2 gRPC): /// Hist.GetInterfaceVersion → HistoryService.GetInterfaceVersion -/// Hist.ValidateClientCredential (loop) → HistoryService.ExchangeKey (loop) +/// Hist.ValidateClientCredential (loop) → StorageService.ValidateClientCredential (loop) /// Hist.OpenConnection2 → HistoryService.OpenConnection /// Retr.StartQuery2 → RetrievalService.StartQuery /// Retr.GetNextQueryResultBuffer2 (loop) → RetrievalService.GetNextQueryResultBuffer (loop) /// Retr.EndQuery2 → RetrievalService.EndQuery /// -/// NOTE: not yet live-verified against a 2023 R2 server. The auth handshake uses -/// HistoryService.ExchangeKey because the gRPC HistoryService dropped ValidateClientCredential -/// (it now lives only on StorageService) and gained ExchangeKey with the identical -/// handle+token→token shape. If a live server rejects this, the handshake op is the first thing -/// to revisit — everything else is the proven 2020 byte protocol. +/// LIVE-VERIFIED 2026-06-21 against a real 2023 R2 server (interface versions: History=12, +/// Retrieval=4, Storage=4). The SSPI/Negotiate token loop maps to +/// StorageService.ValidateClientCredential(Handle, InBuff)→(status, OutBuff) — the op that +/// kept the 2020 inBuff/outBuff token framing. The gRPC HistoryService dropped +/// ValidateClientCredential and gained ExchangeKey, but ExchangeKey is a SEPARATE +/// key-exchange/cert-path op, NOT the Negotiate loop: feeding it an NTLM token is rejected at +/// round 0 regardless of credentials. An earlier revision wrongly routed the loop to ExchangeKey; +/// routing it to StorageService.ValidateClientCredential completes the full read chain. The byte +/// payloads (OpenConnection3 v6, token framing, DataQueryRequest, row buffers) are the proven 2020 +/// protocol and transfer unchanged — only the History interface integer differs (12 vs the 2020 +/// value 11), and that version is buffer-compatible (a live read returns rows). /// internal sealed class HistorianGrpcReadOrchestrator { @@ -167,15 +174,16 @@ internal sealed class HistorianGrpcReadOrchestrator new GrpcHistory.GetInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); HistorianServerVersionGate.Validate(HistorianServiceInterface.History, historyVersion.UiVersion, _options); + var storageClient = new GrpcStorage.StorageService.StorageServiceClient(connection.Channel); HistorianNativeHandshake.RunTokenRounds( (handle, wrapped, _) => { - GrpcHistory.ExchangeKeyResponse response = historyClient.ExchangeKey( - new GrpcHistory.ExchangeKeyRequest { StrHandle = handle, BtInput = ByteString.CopyFrom(wrapped) }, + GrpcStorage.ValidateClientCredentialResponse response = storageClient.ValidateClientCredential( + new GrpcStorage.ValidateClientCredentialRequest { Handle = handle, InBuff = ByteString.CopyFrom(wrapped) }, connection.Metadata, Deadline(), cancellationToken); - byte[] serverOutput = response.BtOutput?.ToByteArray() ?? []; + byte[] serverOutput = response.OutBuff?.ToByteArray() ?? []; byte[] error = response.Status?.BtError?.ToByteArray() ?? []; bool success = response.Status?.BSuccess ?? false; return new HistorianNativeHandshake.TokenExchangeResult(success, serverOutput, error); diff --git a/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs b/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs index aa05f3c..ba9f984 100644 --- a/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs +++ b/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs @@ -43,6 +43,15 @@ internal static class HistorianServerVersionGate public const uint RetrievalInterfaceVersion = 4; public const uint TransactionInterfaceVersion = 2; + /// + /// The 2023 R2 gRPC HistoryService reports interface version 12. It is buffer-compatible with + /// the 2020 version 11 — the OpenConnection3 v6 / token / DataQueryRequest / row buffers are + /// byte-identical — confirmed by a live end-to-end gRPC read against a real 2023 R2 server + /// (2026-06-21). So both 11 and 12 are accepted for History. (Retrieval reported 4, matching + /// the 2020 value, so it needs no widening.) + /// + public const uint HistoryInterfaceVersionGrpc2023R2 = 12; + /// /// True when the service interface reports a meaningful version that should be matched. /// Status is reachability-only (its GetInterfaceVersion returns 0). @@ -56,7 +65,7 @@ internal static class HistorianServerVersionGate _ => false }; - /// The interface version this SDK's serializers target for a value-gated service. + /// The canonical interface version this SDK's serializers target for a value-gated service. public static uint ExpectedVersion(HistorianServiceInterface service) => service switch { HistorianServiceInterface.History => HistoryInterfaceVersion, @@ -65,6 +74,18 @@ internal static class HistorianServerVersionGate _ => throw new ArgumentOutOfRangeException(nameof(service), service, "Service interface is not value-gated.") }; + /// + /// All interface versions accepted for a value-gated service. Usually a single value, but + /// History accepts both the 2020 value (11) and the buffer-compatible 2023 R2 gRPC value (12). + /// + public static uint[] AcceptedVersions(HistorianServiceInterface service) => service switch + { + HistorianServiceInterface.History => [HistoryInterfaceVersion, HistoryInterfaceVersionGrpc2023R2], + HistorianServiceInterface.Retrieval => [RetrievalInterfaceVersion], + HistorianServiceInterface.Transaction => [TransactionInterfaceVersion], + _ => throw new ArgumentOutOfRangeException(nameof(service), service, "Service interface is not value-gated.") + }; + /// /// Throws when version verification is enabled /// and the server's reported interface version differs from the version this SDK targets. @@ -80,14 +101,15 @@ internal static class HistorianServerVersionGate return; } - uint expected = ExpectedVersion(service); - if (reportedVersion == expected) + uint[] accepted = AcceptedVersions(service); + if (Array.IndexOf(accepted, reportedVersion) >= 0) { return; } + string acceptedList = string.Join(", ", accepted); throw new ProtocolEvidenceMissingException( - $"{service} interface version {reportedVersion} (this SDK's serializers target version {expected}); " + + $"{service} interface version {reportedVersion} (this SDK's serializers target version {acceptedList}); " + $"set {nameof(HistorianClientOptions)}.{nameof(HistorianClientOptions.VerifyServerInterfaceVersion)}=false to bypass at your own risk"); } } diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs new file mode 100644 index 0000000..7958f28 --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcHandshakeRoutingTests.cs @@ -0,0 +1,153 @@ +using System.Reflection; +using System.Reflection.Emit; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Structural guardrail pinning the 2023 R2 gRPC auth-handshake op routing. The SSPI/Negotiate +/// token loop MUST be carried by StorageService.ValidateClientCredential (the op that kept +/// the 2020 inBuff/outBuff token framing), NOT by HistoryService.ExchangeKey — ExchangeKey +/// is a separate key-exchange/cert-path op that rejects an NTLM token at round 0 regardless of +/// credentials (live-confirmed against a real 2023 R2 server, 2026-06-21). An earlier revision +/// routed the loop to ExchangeKey; this test fails if that regression returns. +/// +/// It works by disassembling the IL of HistorianGrpcReadOrchestrator (and its +/// compiler-generated nested closure types — the token-loop call lives inside a lambda) and +/// collecting every method invoked. +/// +public sealed class HistorianGrpcHandshakeRoutingTests +{ + [Fact] + public void Handshake_UsesValidateClientCredential_NotExchangeKey() + { + HashSet calledMethods = CollectCalledMethodNames( + "AVEVA.Historian.Client.Grpc.HistorianGrpcReadOrchestrator"); + + Assert.Contains("ValidateClientCredential", calledMethods); + Assert.DoesNotContain("ExchangeKey", calledMethods); + } + + private static HashSet CollectCalledMethodNames(string typeFullName) + { + Assembly sdk = typeof(HistorianClientOptions).Assembly; + Type orchestrator = sdk.GetType(typeFullName, throwOnError: true)!; + Module module = orchestrator.Module; + + // The orchestrator type plus its compiler-generated nested types (lambda closures). + IEnumerable types = new[] { orchestrator } + .Concat(orchestrator.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic)); + + var names = new HashSet(StringComparer.Ordinal); + foreach (Type t in types) + { + Type[] typeArgs = t.IsGenericType ? t.GetGenericArguments() : Type.EmptyTypes; + foreach (MethodInfo m in t.GetMethods( + BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly)) + { + byte[]? il = m.GetMethodBody()?.GetILAsByteArray(); + if (il is null) + { + continue; + } + + Type[] methodArgs = m.IsGenericMethodDefinition ? m.GetGenericArguments() : Type.EmptyTypes; + foreach (int token in EnumerateMethodTokens(il)) + { + try + { + MethodBase? resolved = module.ResolveMethod(token, typeArgs, methodArgs); + if (resolved is not null) + { + names.Add(resolved.Name); + } + } + catch (Exception ex) when (ex is ArgumentException or BadImageFormatException) + { + // vararg / MethodSpec tokens that don't resolve cleanly — irrelevant here. + } + } + } + } + + return names; + } + + /// + /// Walks an IL byte stream, yielding the 4-byte metadata token of every call/callvirt/newobj + /// (any opcode). Uses the reflection-emit opcode table so + /// operands of other instructions are skipped correctly rather than misread as opcodes. + /// + private static IEnumerable EnumerateMethodTokens(byte[] il) + { + int pos = 0; + while (pos < il.Length) + { + OpCode op; + if (il[pos] == 0xFE && pos + 1 < il.Length) + { + op = TwoByteOpCodes[il[pos + 1]]; + pos += 2; + } + else + { + op = OneByteOpCodes[il[pos]]; + pos += 1; + } + + switch (op.OperandType) + { + case OperandType.InlineMethod: + yield return BitConverter.ToInt32(il, pos); + pos += 4; + break; + case OperandType.InlineNone: + break; + case OperandType.ShortInlineBrTarget: + case OperandType.ShortInlineI: + case OperandType.ShortInlineVar: + pos += 1; + break; + case OperandType.InlineVar: + pos += 2; + break; + case OperandType.InlineI8: + case OperandType.InlineR: + pos += 8; + break; + case OperandType.InlineSwitch: + int count = BitConverter.ToInt32(il, pos); + pos += 4 + (4 * count); + break; + default: + // InlineBrTarget, InlineField, InlineI, InlineSig, InlineString, InlineTok, + // InlineType, ShortInlineR — all 4-byte operands. + pos += 4; + break; + } + } + } + + private static readonly OpCode[] OneByteOpCodes = BuildOpCodeTable(twoByte: false); + private static readonly OpCode[] TwoByteOpCodes = BuildOpCodeTable(twoByte: true); + + private static OpCode[] BuildOpCodeTable(bool twoByte) + { + var table = new OpCode[256]; + foreach (FieldInfo f in typeof(OpCodes).GetFields(BindingFlags.Public | BindingFlags.Static)) + { + if (f.GetValue(null) is not OpCode op) + { + continue; + } + + ushort value = unchecked((ushort)op.Value); + bool isTwoByte = (value & 0xFF00) == 0xFE00; + if (isTwoByte == twoByte) + { + table[value & 0xFF] = op; + } + } + + return table; + } +} diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs index 9c3a98e..cbc7539 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs @@ -29,6 +29,20 @@ public sealed class HistorianServerVersionGateTests HistorianServerVersionGate.Validate(HistorianServiceInterface.Transaction, HistorianServerVersionGate.TransactionInterfaceVersion, Options()); } + [Fact] + public void Validate_History_AcceptsBoth2020And2023R2Versions() + { + // History 11 (2020 WCF) and 12 (2023 R2 gRPC) are both buffer-compatible — a live gRPC + // read against a real 2023 R2 server (interface version 12) returns rows. Both must pass. + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, 11u, Options()); + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, HistorianServerVersionGate.HistoryInterfaceVersionGrpc2023R2, Options()); + Assert.Equal(12u, HistorianServerVersionGate.HistoryInterfaceVersionGrpc2023R2); + Assert.Contains(11u, HistorianServerVersionGate.AcceptedVersions(HistorianServiceInterface.History)); + Assert.Contains(12u, HistorianServerVersionGate.AcceptedVersions(HistorianServiceInterface.History)); + // Retrieval reported 4 on the live 2023 R2 server — matches 2020, so it is NOT widened. + Assert.DoesNotContain(5u, HistorianServerVersionGate.AcceptedVersions(HistorianServiceInterface.Retrieval)); + } + [Fact] public void Validate_MismatchedVersion_ThrowsProtocolEvidenceMissing() { @@ -36,7 +50,7 @@ public sealed class HistorianServerVersionGateTests (HistorianServiceInterface Service, uint Version)[] cases = [ (HistorianServiceInterface.History, 10u), - (HistorianServiceInterface.History, 12u), + (HistorianServiceInterface.History, 13u), (HistorianServiceInterface.Retrieval, 3u), (HistorianServiceInterface.Retrieval, 5u), (HistorianServiceInterface.Transaction, 1u),