From 1e9a87fce91258b64acb0526554f90b64f60b7c7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 14:27:47 -0400 Subject: [PATCH 1/6] Add 2023 R2 gRPC transport (RemoteGrpc) reusing native byte payloads Stands up HistorianTransport.RemoteGrpc end-to-end for the read path, built on the recovered 2023 R2 gRPC contract (gRPC-Web/HTTP-1.1, port 32565, gzip). The opaque protobuf `bytes` fields carry the SAME native binary payloads as the 2020 WCF/MDAS path, so the proven serializers and parsers are reused unchanged. - Grpc/Protos/*.proto: 6 protoc-validated contracts recovered from embedded FileDescriptors (authoritative, not guessed). - Grpc/HistorianGrpcChannelFactory: GrpcWebHandler/HTTP-1.1 channel, ResolvePort/ResolveAddress, optional TLS + gzip. - Grpc/HistorianGrpcReadOrchestrator: mirrors the WCF read chain over gRPC; auth uses HistoryService.ExchangeKey (the gRPC ValCl op). - Wcf/HistorianNativeHandshake: transport-agnostic Open2 request builder + SSPI/Negotiate token loop + response decode, shared by WCF and gRPC. - Op map (2020 -> gRPC): ValCl->ExchangeKey, Open2->OpenConnection, StartQuery2->StartQuery, GetNextQueryResultBuffer2->GetNextQueryResultBuffer. - HistorianClientOptions: DefaultGrpcPort=32565, GrpcUseTls. - csproj: Google.Protobuf, Grpc.Net.Client(.Web), Grpc.Tools codegen. Not yet live-verified against a 2023 R2 server: ExchangeKey is the first thing to revisit if a live server rejects the handshake; the inner byte payloads are the proven 2020 protocol. Gated live test via HISTORIAN_GRPC_HOST. 188 unit tests green; build clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 1 + .../AVEVA.Historian.Client.csproj | 17 + .../Grpc/HistorianGrpcChannelFactory.cs | 92 ++++ .../Grpc/HistorianGrpcReadOrchestrator.cs | 363 +++++++++++++++ .../Grpc/Protos/HistoryService.proto | 209 +++++++++ .../Grpc/Protos/RetrievalService.proto | 186 ++++++++ .../Grpc/Protos/Status.proto | 12 + .../Grpc/Protos/StatusService.proto | 215 +++++++++ .../Grpc/Protos/StorageService.proto | 417 ++++++++++++++++++ .../Grpc/Protos/TransactionService.proto | 92 ++++ .../HistorianClientOptions.cs | 13 + .../HistorianTransport.cs | 9 +- .../Protocol/Historian2020ProtocolDialect.cs | 18 +- .../Wcf/HistorianNativeHandshake.cs | 165 +++++++ .../Wcf/HistorianWcfAuthChainHelper.cs | 116 +---- .../Wcf/HistorianWcfReadOrchestrator.cs | 6 +- .../HistorianGrpcIntegrationTests.cs | 63 +++ .../HistorianGrpcTransportTests.cs | 114 +++++ 18 files changed, 1991 insertions(+), 117 deletions(-) create mode 100644 src/AVEVA.Historian.Client/Grpc/HistorianGrpcChannelFactory.cs create mode 100644 src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/HistoryService.proto create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/RetrievalService.proto create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/Status.proto create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/StatusService.proto create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/StorageService.proto create mode 100644 src/AVEVA.Historian.Client/Grpc/Protos/TransactionService.proto create mode 100644 src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs create mode 100644 tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs create mode 100644 tests/AVEVA.Historian.Client.Tests/HistorianGrpcTransportTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 93e6ee3..92f1e90 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,7 @@ Three layered subsystems, intentionally decoupled so protocol parsing can be uni - **`Wcf/`** — managed WCF/MDAS layer. The Historian uses Net.TCP on port `32568` with a custom `application/x-mdas` content type wrapping a binary SOAP 1.2 / WS-Addressing 1.0 envelope. `MdasMessageEncoder` + `MdasMessageEncodingBindingElement` implement that wrapper. `HistorianWcfBindingFactory` produces three flavors: plain MDAS, MDAS+Windows transport (used for `/Hist-Integrated`), and MDAS+certificate (used for `/HistCert`). Service paths live in `HistorianWcfServiceNames`. WCF data contracts (`Wcf/Contracts/`) are reproduced from server-side static analysis and are versioned per native interface (e.g., `IRetrievalServiceContract2..4`). - **`Protocol/`** — binary frame layer (`HistorianFrameReader`/`Writer`, `HistorianBinaryPrimitives`, `HistorianMessageType`). `Historian2020ProtocolDialect` is the version-anchored bridge between `HistorianClient` and the frame layer; methods without sufficient evidence throw `ProtocolEvidenceMissingException` rather than guessing wire bytes. - **`Transport/`** — pluggable `IHistorianTransport` (default: TCP). Tests inject a fake transport. +- **`Grpc/`** — 2023 R2 gRPC transport (`HistorianTransport.RemoteGrpc`). The recovered protobuf contract lives in `Grpc/Protos/*.proto` and is compiled to client stubs at build time by `Grpc.Tools`. `HistorianGrpcChannelFactory` builds a gRPC-Web/HTTP-1.1 channel (default port `32565`, optional TLS, gzip) matching the stock 2023 R2 client. `HistorianGrpcReadOrchestrator` mirrors `HistorianWcfReadOrchestrator` but over gRPC: it reuses the exact native serializers/parsers — the same Open2 buffer, SSPI/NTLM tokens, and `DataQueryRequest`/result buffers travel inside protobuf `bytes` fields. The 2020→gRPC op map: `Hist.ValCl`→`HistoryService.ExchangeKey`, `Hist.Open2`→`HistoryService.OpenConnection`, `Retr.StartQuery2`→`RetrievalService.StartQuery`, `Retr.GetNextQueryResultBuffer2`→`RetrievalService.GetNextQueryResultBuffer`. The transport-agnostic handshake (Open2 request builder + SSPI token loop + response decode) is shared via `Wcf/HistorianNativeHandshake`. **Not yet live-verified against a 2023 R2 server** — the auth handshake op (`ExchangeKey`) is the first thing to revisit if a live server rejects it; the byte payloads are the proven 2020 protocol. Gated live test: set `HISTORIAN_GRPC_HOST` (+ `HISTORIAN_TEST_TAG`, optional `HISTORIAN_GRPC_PORT`/`HISTORIAN_GRPC_TLS`/`HISTORIAN_GRPC_DNSID`). - **`Models/`** — public DTOs and enums (`HistorianSample`, `RetrievalMode`, etc.). `HistorianDataValue` represents the discriminated value type. `InternalsVisibleTo` exposes internals to the test assembly and the reverse-engineering tool. diff --git a/src/AVEVA.Historian.Client/AVEVA.Historian.Client.csproj b/src/AVEVA.Historian.Client/AVEVA.Historian.Client.csproj index be55c84..405d5d6 100644 --- a/src/AVEVA.Historian.Client/AVEVA.Historian.Client.csproj +++ b/src/AVEVA.Historian.Client/AVEVA.Historian.Client.csproj @@ -12,6 +12,23 @@ + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + <_Parameter1>AVEVA.Historian.Client.Tests diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcChannelFactory.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcChannelFactory.cs new file mode 100644 index 0000000..151033b --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcChannelFactory.cs @@ -0,0 +1,92 @@ +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using Grpc.Core; +using Grpc.Net.Client; +using Grpc.Net.Client.Web; + +namespace AVEVA.Historian.Client.Grpc; + +/// +/// Builds a for the 2023 R2 Historian Client Access Point, +/// replicating the stock Archestra.Historian.GrpcClient.GrpcClientBase.InitializeBase +/// transport shape: gRPC-Web (binary) over HTTP/1.1, optional TLS with an +/// untrusted-certificate bypass, and gzip request encoding. +/// +internal static class HistorianGrpcChannelFactory +{ + /// + /// Resolves the effective gRPC port: when the caller left + /// at the WCF default (32568), the 2023 R2 gRPC default (32565) is substituted; otherwise the + /// explicit value is honoured. + /// + internal static int ResolvePort(HistorianClientOptions options) => + options.Port == HistorianClientOptions.DefaultPort ? HistorianClientOptions.DefaultGrpcPort : options.Port; + + /// + /// Builds the channel address. TLS uses https://{ServerDnsIdentity|Host}:{port} (the + /// DNS-identity override lets the URL match the server certificate name when connecting by IP); + /// plaintext uses http://{Host}:{port}. + /// + internal static string ResolveAddress(HistorianClientOptions options) + { + int port = ResolvePort(options); + if (options.GrpcUseTls) + { + string tlsHost = !string.IsNullOrEmpty(options.ServerDnsIdentity) ? options.ServerDnsIdentity! : options.Host; + return $"https://{tlsHost}:{port}"; + } + + return $"http://{options.Host}:{port}"; + } + + public static HistorianGrpcConnection Create(HistorianClientOptions options) + { + string address = ResolveAddress(options); + + var httpHandler = new HttpClientHandler(); + if (options.AllowUntrustedServerCertificate) + { + httpHandler.ServerCertificateCustomValidationCallback = AcceptAnyCertificate; + } + + // gRPC-Web binary mode over HTTP/1.1 — matches the stock client (GrpcWebMode.GrpcWeb, + // HttpVersion 1.1). The 2023 R2 HCAP endpoint speaks gRPC-Web, not bare HTTP/2 gRPC. + var webHandler = new GrpcWebHandler(GrpcWebMode.GrpcWeb, httpHandler) + { + HttpVersion = new Version(1, 1) + }; + + var channelOptions = new GrpcChannelOptions + { + HttpHandler = webHandler + }; + + GrpcChannel channel = GrpcChannel.ForAddress(address, channelOptions); + + // The stock client always advertises gzip request encoding; honour the option so + // bandwidth-limited links can disable it. + var metadata = new Metadata(); + if (options.Compression) + { + metadata.Add("grpc-internal-encoding-request", "gzip"); + } + + return new HistorianGrpcConnection(channel, metadata); + } + + private static bool AcceptAnyCertificate( + HttpRequestMessage request, + X509Certificate2? certificate, + X509Chain? chain, + SslPolicyErrors errors) => true; +} + +/// A live gRPC channel plus the per-call metadata header set. +internal sealed class HistorianGrpcConnection(GrpcChannel channel, Metadata metadata) : IDisposable +{ + public GrpcChannel Channel { get; } = channel; + + public Metadata Metadata { get; } = metadata; + + public void Dispose() => Channel.Dispose(); +} diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs new file mode 100644 index 0000000..d37241c --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs @@ -0,0 +1,363 @@ +using System.Runtime.CompilerServices; +using Google.Protobuf; +using Grpc.Core; +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Wcf; +using GrpcHistory = ArchestrA.Grpc.Contract.History; +using GrpcRetrieval = ArchestrA.Grpc.Contract.Retrieval; + +namespace AVEVA.Historian.Client.Grpc; + +/// +/// 2023 R2 gRPC read orchestrator. Mirrors over the +/// gRPC transport: the same native binary buffers travel inside protobuf bytes fields, +/// and the same serializers/parsers (, +/// ) are reused unchanged. +/// +/// Operation mapping (2020 WCF → 2023 R2 gRPC): +/// Hist.GetInterfaceVersion → HistoryService.GetInterfaceVersion +/// Hist.ValidateClientCredential (loop) → HistoryService.ExchangeKey (loop) +/// Hist.OpenConnection2 → HistoryService.OpenConnection +/// Retr.StartQuery2 → RetrievalService.StartQuery +/// Retr.GetNextQueryResultBuffer2 (loop) → RetrievalService.GetNextQueryResultBuffer (loop) +/// Retr.EndQuery2 → RetrievalService.EndQuery +/// +/// NOTE: not yet live-verified against a 2023 R2 server. The auth handshake uses +/// HistoryService.ExchangeKey because the gRPC HistoryService dropped ValidateClientCredential +/// (it now lives only on StorageService) and gained ExchangeKey with the identical +/// handle+token→token shape. If a live server rejects this, the handshake op is the first thing +/// to revisit — everything else is the proven 2020 byte protocol. +/// +internal sealed class HistorianGrpcReadOrchestrator +{ + private const ushort StartQueryRequestType = HistorianDataQueryProtocol.QueryRequestTypeData; + + private readonly HistorianClientOptions _options; + + public HistorianGrpcReadOrchestrator(HistorianClientOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public async IAsyncEnumerable ReadRawAsync( + string tag, + DateTime startUtc, + DateTime endUtc, + int maxValues, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ValidateAuth(); + cancellationToken.ThrowIfCancellationRequested(); + + IReadOnlyList rows = await Task.Run( + () => RunRawChain(tag, startUtc, endUtc, maxValues, cancellationToken), cancellationToken).ConfigureAwait(false); + foreach (HistorianSample sample in rows) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return sample; + } + } + + public async IAsyncEnumerable ReadAggregateAsync( + string tag, + DateTime startUtc, + DateTime endUtc, + RetrievalMode mode, + TimeSpan interval, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ValidateAuth(); + cancellationToken.ThrowIfCancellationRequested(); + + IReadOnlyList rows = await Task.Run( + () => RunAggregateChain(tag, startUtc, endUtc, mode, interval, cancellationToken), cancellationToken).ConfigureAwait(false); + foreach (HistorianAggregateSample sample in rows) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return sample; + } + } + + public Task> ReadAtTimeAsync( + string tag, + IReadOnlyList timestampsUtc, + CancellationToken cancellationToken) + { + ValidateAuth(); + cancellationToken.ThrowIfCancellationRequested(); + return Task.Run>(() => RunAtTimeChain(tag, timestampsUtc, cancellationToken), cancellationToken); + } + + private void ValidateAuth() + { + if (!_options.IntegratedSecurity && string.IsNullOrEmpty(_options.UserName)) + { + throw new ProtocolEvidenceMissingException( + "Managed gRPC read flow currently requires IntegratedSecurity or an explicit UserName + Password."); + } + } + + private List RunRawChain(string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken cancellationToken) + { + using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); + uint clientHandle = OpenAuthenticatedConnection(connection, cancellationToken); + HistorianDataQueryRequest request = HistorianWcfReadOrchestrator.BuildDataQueryRequest(tag, startUtc, endUtc, maxValues); + return RunQuery(connection, clientHandle, request, maxValues, cancellationToken); + } + + private List RunAggregateChain( + string tag, DateTime startUtc, DateTime endUtc, RetrievalMode mode, TimeSpan interval, CancellationToken cancellationToken) + { + using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); + uint clientHandle = OpenAuthenticatedConnection(connection, cancellationToken); + return RunAggregateQuery(connection, clientHandle, tag, startUtc, endUtc, mode, interval, cancellationToken); + } + + private List RunAtTimeChain(string tag, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) + { + if (timestampsUtc.Count == 0) + { + return []; + } + + using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(_options); + uint clientHandle = OpenAuthenticatedConnection(connection, cancellationToken); + + List results = new(timestampsUtc.Count); + foreach (DateTime ts in timestampsUtc) + { + cancellationToken.ThrowIfCancellationRequested(); + DateTime tsUtc = ts.ToUniversalTime(); + List aggregates = RunAggregateQuery( + connection, + clientHandle, + tag, + tsUtc - TimeSpan.FromTicks(1), + tsUtc + TimeSpan.FromTicks(1), + RetrievalMode.Interpolated, + TimeSpan.FromTicks(2), + cancellationToken); + + if (aggregates.Count == 0) + { + continue; + } + + HistorianAggregateSample chosen = aggregates[0]; + results.Add(new HistorianSample( + TagName: chosen.TagName, + TimestampUtc: tsUtc, + NumericValue: chosen.Value, + StringValue: null, + Quality: chosen.Quality, + QualityDetail: chosen.QualityDetail, + OpcQuality: chosen.OpcQuality, + PercentGood: 100)); + } + + return results; + } + + private uint OpenAuthenticatedConnection(HistorianGrpcConnection connection, CancellationToken cancellationToken) + { + Guid contextKey = Guid.NewGuid(); + var historyClient = new GrpcHistory.HistoryService.HistoryServiceClient(connection.Channel); + + historyClient.GetInterfaceVersion(new GrpcHistory.GetInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); + + HistorianNativeHandshake.RunTokenRounds( + (handle, wrapped, _) => + { + GrpcHistory.ExchangeKeyResponse response = historyClient.ExchangeKey( + new GrpcHistory.ExchangeKeyRequest { StrHandle = handle, BtInput = ByteString.CopyFrom(wrapped) }, + connection.Metadata, + Deadline(), + cancellationToken); + byte[] serverOutput = response.BtOutput?.ToByteArray() ?? []; + byte[] error = response.Status?.BtError?.ToByteArray() ?? []; + bool success = response.Status?.BSuccess ?? false; + return new HistorianNativeHandshake.TokenExchangeResult(success, serverOutput, error); + }, + contextKey, + _options, + cancellationToken); + + byte[] open2Request = HistorianNativeHandshake.BuildOpenConnection3Request( + _options.Host, contextKey, HistorianWcfAuthChainHelper.NativeIntegratedReadOnlyConnectionMode); + + GrpcHistory.OpenConnectionResponse open2 = historyClient.OpenConnection( + new GrpcHistory.OpenConnectionRequest { BtConnectionRequest = ByteString.CopyFrom(open2Request) }, + connection.Metadata, + Deadline(), + cancellationToken); + + byte[] open2Response = open2.BtConnectionResponse?.ToByteArray() ?? []; + if (!(open2.Status?.BSuccess ?? false)) + { + byte[] err = open2.Status?.BtError?.ToByteArray() ?? []; + throw new InvalidOperationException($"gRPC OpenConnection failed (errorLen={err.Length}, responseLen={open2Response.Length})."); + } + + (uint clientHandle, _) = HistorianNativeHandshake.ParseOpenConnectionResponse(open2Response); + return clientHandle; + } + + private List RunQuery( + HistorianGrpcConnection connection, + uint clientHandle, + HistorianDataQueryRequest request, + int maxValues, + CancellationToken cancellationToken) + { + var retrievalClient = new GrpcRetrieval.RetrievalService.RetrievalServiceClient(connection.Channel); + retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + + byte[] requestBuffer = HistorianDataQueryProtocol.SerializeFullHistoryRequest(request); + uint queryHandle = StartQuery(retrievalClient, clientHandle, requestBuffer, "raw", cancellationToken); + + try + { + List samples = []; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + (byte[] resultBuffer, byte[] errorBuffer) = GetNextResultBuffer(retrievalClient, clientHandle, queryHandle, "raw", cancellationToken); + + if (!HistorianDataQueryProtocol.TryParseGetNextQueryResultBufferRows(resultBuffer, errorBuffer, out IReadOnlyList rows, out bool hasMoreData)) + { + throw new InvalidOperationException($"gRPC GetNextQueryResultBuffer returned an unparsable result buffer (length={resultBuffer.Length})."); + } + + foreach (HistorianSample sample in rows) + { + samples.Add(sample); + if (samples.Count >= maxValues) + { + return samples; + } + } + + if (!hasMoreData) + { + return samples; + } + } + } + finally + { + EndQuerySafely(retrievalClient, clientHandle, queryHandle); + } + } + + private List RunAggregateQuery( + HistorianGrpcConnection connection, + uint clientHandle, + string tag, + DateTime startUtc, + DateTime endUtc, + RetrievalMode mode, + TimeSpan interval, + CancellationToken cancellationToken) + { + var retrievalClient = new GrpcRetrieval.RetrievalService.RetrievalServiceClient(connection.Channel); + retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + + HistorianDataQueryRequest request = HistorianWcfReadOrchestrator.BuildAggregateQueryRequest(tag, startUtc, endUtc, mode, interval); + byte[] requestBuffer = HistorianDataQueryProtocol.SerializeFullHistoryRequest(request); + uint queryHandle = StartQuery(retrievalClient, clientHandle, requestBuffer, $"aggregate {mode}", cancellationToken); + + try + { + List samples = []; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + (byte[] resultBuffer, byte[] errorBuffer) = GetNextResultBuffer(retrievalClient, clientHandle, queryHandle, $"aggregate {mode}", cancellationToken); + + if (!HistorianDataQueryProtocol.TryParseGetNextQueryResultBufferAggregateRows( + resultBuffer, errorBuffer, mode, interval, out IReadOnlyList rows, out bool hasMoreData)) + { + throw new InvalidOperationException($"gRPC GetNextQueryResultBuffer (aggregate {mode}) returned an unparsable buffer (length={resultBuffer.Length})."); + } + + samples.AddRange(rows); + if (!hasMoreData) + { + return samples; + } + } + } + finally + { + EndQuerySafely(retrievalClient, clientHandle, queryHandle); + } + } + + private uint StartQuery( + GrpcRetrieval.RetrievalService.RetrievalServiceClient client, + uint clientHandle, + byte[] requestBuffer, + string label, + CancellationToken cancellationToken) + { + GrpcRetrieval.StartQueryResponse response = client.StartQuery( + new GrpcRetrieval.StartQueryRequest + { + UiHandle = clientHandle, + UiQueryRequestType = StartQueryRequestType, + BtRequestBuffer = ByteString.CopyFrom(requestBuffer) + }, + null, + Deadline(), + cancellationToken); + + if (!(response.Status?.BSuccess ?? false)) + { + byte[] err = response.Status?.BtError?.ToByteArray() ?? []; + throw new InvalidOperationException($"gRPC StartQuery ({label}) failed (errorLen={err.Length})."); + } + + return response.UiQueryHandle; + } + + private (byte[] ResultBuffer, byte[] ErrorBuffer) GetNextResultBuffer( + GrpcRetrieval.RetrievalService.RetrievalServiceClient client, + uint clientHandle, + uint queryHandle, + string label, + CancellationToken cancellationToken) + { + GrpcRetrieval.GetNextQueryResultBufferResponse response = client.GetNextQueryResultBuffer( + new GrpcRetrieval.GetNextQueryResultBufferRequest { UiHandle = clientHandle, UiQueryHandle = queryHandle }, + null, + Deadline(), + cancellationToken); + + byte[] errorBuffer = response.Status?.BtError?.ToByteArray() ?? []; + if (!(response.Status?.BSuccess ?? false)) + { + throw new InvalidOperationException($"gRPC GetNextQueryResultBuffer ({label}) failed (errorLen={errorBuffer.Length})."); + } + + byte[] resultBuffer = response.BtQueryResult?.ToByteArray() ?? []; + return (resultBuffer, errorBuffer); + } + + private void EndQuerySafely(GrpcRetrieval.RetrievalService.RetrievalServiceClient client, uint clientHandle, uint queryHandle) + { + try + { + client.EndQuery( + new GrpcRetrieval.EndQueryRequest { UiHandle = clientHandle, UiQueryHandle = queryHandle }, + null, + Deadline(), + CancellationToken.None); + } + catch + { + // Best-effort cleanup; the read result is already collected. + } + } + + private DateTime Deadline() => DateTime.UtcNow.Add(_options.RequestTimeout); +} diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/HistoryService.proto b/src/AVEVA.Historian.Client/Grpc/Protos/HistoryService.proto new file mode 100644 index 0000000..5207efe --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/HistoryService.proto @@ -0,0 +1,209 @@ +// Recovered from HistoryService.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + +import "Status.proto"; + +option csharp_namespace = "ArchestrA.Grpc.Contract.History"; + +message CreateTagResponse { + bool bSuccess = 1; + bytes tagid = 2; +} + +message GetInterfaceVersionRequest { +} + +message GetInterfaceVersionResponse { + uint32 uiError = 1; + uint32 uiVersion = 2; +} + +message OpenConnectionRequest { + bytes btConnectionRequest = 1; +} + +message OpenConnectionResponse { + .Status status = 1; + bytes btConnectionResponse = 2; +} + +message CloseConnectionRequest { + string strHandle = 1; +} + +message CloseConnectionResponse { + .Status status = 1; +} + +message UpdateClientStatusRequest { + string strHandle = 1; + bytes btClientStatus = 2; +} + +message UpdateClientStatusResponse { + .Status status = 1; + bytes btServerStatus = 2; +} + +message RegisterTagsRequest { + string strHandle = 1; + bytes btTagInfos = 2; +} + +message RegisterTagsResponse { + .Status status = 1; + bytes btTagStatus = 2; +} + +message EnsureTagsRequest { + string strHandle = 1; + bytes btTagInfos = 2; + uint32 elementCount = 3; +} + +message EnsureTagsResponse { + .Status status = 1; + bytes btTagStatus = 2; +} + +message AddStreamValuesRequest { + string strHandle = 1; + bytes btValues = 2; +} + +message AddStreamValuesResponse { + .Status status = 1; +} + +message TagExtendedProperty { + enum TagExtendedPropertyDataType { + String = 0; + Int16 = 1; + Int32 = 2; + Int64 = 3; + Double = 4; + Boolean = 5; + DateTimeOffset = 6; + Guid = 7; + Geography = 8; + Geometry = 9; + } + + string PropertyName = 1; + .TagExtendedProperty.TagExtendedPropertyDataType type = 2; + bytes value = 3; + bool Facetable = 4; + bool Searchable = 5; + bool SubstringSearchable = 6; +} + +message TagExtendedPropertyGroup { + string tagname = 1; + repeated .TagExtendedProperty TagExtendedProperties = 2; +} + +message AddTagExtendedPropertyRequest { + string strHandle = 1; + repeated .TagExtendedPropertyGroup TagExtendedPropertyGroups = 2; +} + +message AddTagExtendedPropertyResponse { + .Status status = 1; +} + +message ExchangeKeyRequest { + string strHandle = 1; + bytes btInput = 2; +} + +message ExchangeKeyResponse { + .Status status = 1; + bytes btOutput = 2; +} + +message StartJobRequest { + string strHandle = 1; + bytes btInput = 2; +} + +message StartJobResponse { + .Status status = 1; + string strJobid = 2; +} + +message GetJobStatusRequest { + string strHandle = 1; + string strJobid = 2; +} + +message GetJobStatusResponse { + .Status status = 1; + bytes btJobStatus = 2; +} + +message AddTagExtendedPropertiesRequest { + string strHandle = 1; + bytes btTeps = 2; +} + +message AddTagExtendedPropertiesResponse { + .Status status = 1; +} + +message DeleteTagExtendedPropertiesRequest { + string strHandle = 1; + bytes btInput = 2; +} + +message DeleteTagExtendedPropertiesResponse { + .Status status = 1; +} + +message DeleteTagsRequest { + uint32 uiHandle = 1; + bytes btTagnames = 2; +} + +message DeleteTagsResponse { + .Status status = 1; + bytes btDeleteTagStatus = 2; +} + +message AddTagLocalizedPropertiesRequest { + string strHandle = 1; + bytes btInput = 2; +} + +message AddTagLocalizedPropertiesResponse { + .Status status = 1; +} + +message DeleteTagLocalizedPropertiesRequest { + string strHandle = 1; + bytes btInput = 2; +} + +message DeleteTagLocalizedPropertiesResponse { + .Status status = 1; +} + +service HistoryService { + rpc GetInterfaceVersion (.GetInterfaceVersionRequest) returns (.GetInterfaceVersionResponse); + rpc ExchangeKey (.ExchangeKeyRequest) returns (.ExchangeKeyResponse); + rpc OpenConnection (.OpenConnectionRequest) returns (.OpenConnectionResponse); + rpc CloseConnection (.CloseConnectionRequest) returns (.CloseConnectionResponse); + rpc UpdateClientStatus (.UpdateClientStatusRequest) returns (.UpdateClientStatusResponse); + rpc RegisterTags (.RegisterTagsRequest) returns (.RegisterTagsResponse); + rpc EnsureTags (.EnsureTagsRequest) returns (.EnsureTagsResponse); + rpc AddStreamValues (.AddStreamValuesRequest) returns (.AddStreamValuesResponse); + rpc AddTagExtendedPropertyGroups (.AddTagExtendedPropertyRequest) returns (.AddTagExtendedPropertyResponse); + rpc AddTagExtendedProperties (.AddTagExtendedPropertiesRequest) returns (.AddTagExtendedPropertiesResponse); + rpc StartJob (.StartJobRequest) returns (.StartJobResponse); + rpc GetJobStatus (.GetJobStatusRequest) returns (.GetJobStatusResponse); + rpc DeleteTagExtendedProperties (.DeleteTagExtendedPropertiesRequest) returns (.DeleteTagExtendedPropertiesResponse); + rpc DeleteTags (.DeleteTagsRequest) returns (.DeleteTagsResponse); + rpc AddTagLocalizedProperties (.AddTagLocalizedPropertiesRequest) returns (.AddTagLocalizedPropertiesResponse); + rpc DeleteTagLocalizedProperties (.DeleteTagLocalizedPropertiesRequest) returns (.DeleteTagLocalizedPropertiesResponse); +} + diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/RetrievalService.proto b/src/AVEVA.Historian.Client/Grpc/Protos/RetrievalService.proto new file mode 100644 index 0000000..8f50c13 --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/RetrievalService.proto @@ -0,0 +1,186 @@ +// Recovered from RetrievalService.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + +import "Status.proto"; + +option csharp_namespace = "ArchestrA.Grpc.Contract.Retrieval"; + +message GetRetrievalInterfaceVersionRequest { +} + +message GetRetrievalInterfaceVersionResponse { + uint32 uiError = 1; + uint32 uiVersion = 2; +} + +message StartQueryRequest { + uint32 uiHandle = 1; + uint32 uiQueryRequestType = 2; + bytes btRequestBuffer = 3; +} + +message StartQueryResponse { + .Status status = 1; + uint32 uiQueryHandle = 2; + bytes btResponseBuffer = 3; +} + +message GetNextQueryResultBufferRequest { + uint32 uiHandle = 1; + uint32 uiQueryHandle = 2; +} + +message GetNextQueryResultBufferResponse { + .Status status = 1; + bytes btQueryResult = 2; +} + +message EndQueryRequest { + uint32 uiHandle = 1; + uint32 uiQueryHandle = 2; +} + +message EndQueryResponse { + .Status status = 1; +} + +message GetShardTagidsByTagnameAndSourceRequest { + string strHandle = 1; + bytes btTagnameAndSource = 2; +} + +message GetShardTagidsByTagnameAndSourceResponse { + .Status status = 1; + bytes btShardTagids = 2; +} + +message GetTagInfosFromNameRequest { + string strHandle = 1; + bytes btTagNames = 2; + uint32 uiSequence = 3; +} + +message GetTagInfosFromNameResponse { + .Status status = 1; + bytes btTagInfos = 2; + uint32 uiSequence = 3; +} + +message GetTagExtendedPropertiesFromNameRequest { + string strHandle = 1; + bytes btTagNames = 2; + uint32 uiSequence = 3; +} + +message GetTagExtendedPropertiesFromNameResponse { + .Status status = 1; + bytes btTeps = 2; + uint32 uiSequence = 3; +} + +message ExecuteSqlCommandRequest { + string strHandle = 1; + string StrCommand = 2; + uint32 uiOption = 3; + uint32 uiQueryHandle = 4; +} + +message ExecuteSqlCommandResponse { + .Status status = 1; + int32 iRetValue = 2; + uint32 uiQueryHandle = 3; +} + +message StartEventQueryRequest { + uint32 uiHandle = 1; + uint32 uiQueryRequestType = 2; + bytes btRequest = 3; + uint32 uiQueryHandle = 4; +} + +message StartEventQueryResponse { + .Status status = 1; + uint32 uiQueryHandle = 2; + bytes btResonse = 3; +} + +message GetNextEventQueryResultBufferRequest { + uint32 uiHandle = 1; + uint32 uiQueryHandle = 2; +} + +message GetNextEventQueryResultBufferResponse { + .Status status = 1; + bytes btResult = 2; +} + +message EndEventQueryRequest { + uint32 uiHandle = 1; + uint32 uiQueryHandle = 2; +} + +message EndEventQueryResponse { + .Status status = 1; +} + +message StartTagQueryRequest { + string strHandle = 1; + bytes btRequest = 2; +} + +message StartTagQueryResponse { + .Status status = 1; + bytes btResponse = 2; +} + +message QueryTagRequest { + string strHandle = 1; + uint32 uiQueryHandle = 2; + bytes btRequest = 3; +} + +message QueryTagResponse { + .Status status = 1; + bytes btResonse = 2; +} + +message EndTagQueryRequest { + string strHandle = 1; + uint32 uiQueryHandle = 2; +} + +message EndTagQueryResponse { + .Status status = 1; +} + +message GetTagLocalizedPropertiesFromNameRequest { + string strHandle = 1; + bytes btTagNames = 2; + uint32 uiSequence = 3; +} + +message GetTagLocalizedPropertiesFromNameResponse { + .Status status = 1; + uint32 uiSequence = 2; + bytes btOutBuffer = 3; +} + +service RetrievalService { + rpc GetRetrievalInterfaceVersion (.GetRetrievalInterfaceVersionRequest) returns (.GetRetrievalInterfaceVersionResponse); + rpc StartQuery (.StartQueryRequest) returns (.StartQueryResponse); + rpc GetNextQueryResultBuffer (.GetNextQueryResultBufferRequest) returns (.GetNextQueryResultBufferResponse); + rpc EndQuery (.EndQueryRequest) returns (.EndQueryResponse); + rpc GetShardTagidsByTagnameAndSource (.GetShardTagidsByTagnameAndSourceRequest) returns (.GetShardTagidsByTagnameAndSourceResponse); + rpc GetTagInfosFromName (.GetTagInfosFromNameRequest) returns (.GetTagInfosFromNameResponse); + rpc GetTagExtendedPropertiesFromName (.GetTagExtendedPropertiesFromNameRequest) returns (.GetTagExtendedPropertiesFromNameResponse); + rpc ExecuteSqlCommand (.ExecuteSqlCommandRequest) returns (.ExecuteSqlCommandResponse); + rpc StartEventQuery (.StartEventQueryRequest) returns (.StartEventQueryResponse); + rpc GetNextEventQueryResultBuffer (.GetNextEventQueryResultBufferRequest) returns (.GetNextEventQueryResultBufferResponse); + rpc EndEventQuery (.EndEventQueryRequest) returns (.EndEventQueryResponse); + rpc StartTagQuery (.StartTagQueryRequest) returns (.StartTagQueryResponse); + rpc QueryTag (.QueryTagRequest) returns (.QueryTagResponse); + rpc EndTagQuery (.EndTagQueryRequest) returns (.EndTagQueryResponse); + rpc GetTagLocalizedPropertiesFromName (.GetTagLocalizedPropertiesFromNameRequest) returns (.GetTagLocalizedPropertiesFromNameResponse); +} + diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/Status.proto b/src/AVEVA.Historian.Client/Grpc/Protos/Status.proto new file mode 100644 index 0000000..4623094 --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/Status.proto @@ -0,0 +1,12 @@ +// Recovered from Status.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + + +option csharp_namespace = "ArchestrA.Grpc.Contract.RequestStatus"; + +message Status { + bool bSuccess = 1; + bytes btError = 2; +} + diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/StatusService.proto b/src/AVEVA.Historian.Client/Grpc/Protos/StatusService.proto new file mode 100644 index 0000000..6f98388 --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/StatusService.proto @@ -0,0 +1,215 @@ +// Recovered from StatusService.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + +import "Status.proto"; + +option csharp_namespace = "ArchestrA.Grpc.Contract.Status"; + +message GetStatusInterfaceVersionRequest { +} + +message GetStatusInterfaceVersionResponse { + uint32 uiError = 1; + uint32 uiVersion = 2; +} + +message GetSystemParameterRequest { + uint32 uiHandle = 1; + string strParameterName = 2; +} + +message GetSystemParameterResponse { + .Status status = 1; + string strParameterValue = 2; +} + +message SendInfoRequest { + string strHandle = 1; + string strPipeName = 2; + uint32 uiOption = 3; + bytes btReqBuff = 4; + string strInfoID = 5; +} + +message SendInfoResponse { + .Status status = 1; + string strInfoID = 2; + bytes btRespBuff = 3; +} + +message RequestInfoRequest { + string strHandle = 1; + string strInfoID = 2; + uint32 uiOffset = 3; +} + +message RequestInfoResponse { + .Status status = 1; + bytes btRespBuff = 2; +} + +message DeleteInfoRequest { + string strHandle = 1; + string strInfoID = 2; +} + +message DeleteInfoResponse { + .Status status = 1; +} + +message GetHistorianInfoRequest { + string strHandle = 1; + bytes btRequest = 2; +} + +message GetHistorianInfoResponse { + .Status status = 1; + bytes btHistorianInfo = 2; +} + +message StartProcessRequest { + string strHandle = 1; + string strPipeName = 2; + string strPath = 3; + string strAuguments = 4; + uint32 uiKeepAliveInterval = 5; + uint32 uiKeepAliveMethod = 6; +} + +message StartProcessResponse { + .Status status = 1; +} + +message StopProcessRequest { + string strHandle = 1; + string StrPipeName = 2; +} + +message StopProcessResponse { + .Status status = 1; +} + +message PingServerRequest { + string strHandle = 1; + string strPipeName = 2; + uint32 uiTimeout = 3; +} + +message PingServerResponse { + .Status status = 1; +} + +message PingPipeRequest { + string strHandle = 1; + string strPipeName = 2; +} + +message PingPipeResponse { + .Status status = 1; +} + +message ConfigureAutoStartProcessRequest { + string strHandle = 1; + string strPipeName = 2; + string strPath = 3; + string strAuguments = 4; + uint32 uiKeepAliveInterval = 5; + uint32 uiKeepAliveMethod = 6; + uint32 uiStartupFlags = 7; +} + +message ConfigureAutoStartProcessResponse { + .Status status = 1; +} + +message GetHistorianConsoleStatusRequest { + string strHandle = 1; +} + +message GetHistorianConsoleStatusResponse { + .Status status = 1; + uint32 uiConsoleStatus = 2; +} + +message GetRuntimeParameterRequest { + string strHandle = 1; + bytes btRequest = 2; +} + +message GetRuntimeParameterResponse { + .Status status = 1; + bytes btResponse = 2; +} + +message GetSystemTimeZoneNameRequest { + uint32 uiHandle = 1; +} + +message GetSystemTimeZoneNameResponse { + .Status status = 1; + string strSystemTimeZoneName = 2; +} + +message SetHistorianConsoleStatusRequest { + string strHandle = 1; + uint32 uiStatus = 2; + uint32 uiOption = 3; +} + +message SetHistorianConsoleStatusResponse { + .Status status = 1; +} + +message CanUpdateAreaHierarchyRequest { + uint32 uiHandle = 1; +} + +message CanUpdateAreaHierarchyResponse { + .Status status = 1; + bool canUpdate = 2; +} + +message UpdateAreaHierarchyRequest { + uint32 uiHandle = 1; + string guid = 2; + uint32 sequence = 3; + bytes buffer = 4; +} + +message UpdateAreaHierarchyResponse { + .Status status = 1; +} + +message UpdateObjectHierarchyRequest { + uint32 uiHandle = 1; + string guid = 2; + uint32 sequence = 3; + bytes buffer = 4; +} + +message UpdateObjectHierarchyResponse { + .Status status = 1; +} + +service StatusService { + rpc GetStatusInterfaceVersion (.GetStatusInterfaceVersionRequest) returns (.GetStatusInterfaceVersionResponse); + rpc GetSystemParameter (.GetSystemParameterRequest) returns (.GetSystemParameterResponse); + rpc SendInfo (.SendInfoRequest) returns (.SendInfoResponse); + rpc RequestInfo (.RequestInfoRequest) returns (.RequestInfoResponse); + rpc DeleteInfo (.DeleteInfoRequest) returns (.DeleteInfoResponse); + rpc GetHistorianInfo (.GetHistorianInfoRequest) returns (.GetHistorianInfoResponse); + rpc StartProcess (.StartProcessRequest) returns (.StartProcessResponse); + rpc StopProcess (.StopProcessRequest) returns (.StopProcessResponse); + rpc PingServer (.PingServerRequest) returns (.PingServerResponse); + rpc PingPipe (.PingPipeRequest) returns (.PingPipeResponse); + rpc ConfigureAutoStartProcess (.ConfigureAutoStartProcessRequest) returns (.ConfigureAutoStartProcessResponse); + rpc GetHistorianConsoleStatus (.GetHistorianConsoleStatusRequest) returns (.GetHistorianConsoleStatusResponse); + rpc GetRuntimeParameter (.GetRuntimeParameterRequest) returns (.GetRuntimeParameterResponse); + rpc GetSystemTimeZoneName (.GetSystemTimeZoneNameRequest) returns (.GetSystemTimeZoneNameResponse); + rpc SetHistorianConsoleStatus (.SetHistorianConsoleStatusRequest) returns (.SetHistorianConsoleStatusResponse); + rpc CanUpdateAreaHierarchy (.CanUpdateAreaHierarchyRequest) returns (.CanUpdateAreaHierarchyResponse); + rpc UpdateAreaHierarchy (.UpdateAreaHierarchyRequest) returns (.UpdateAreaHierarchyResponse); + rpc UpdateObjectHierarchy (.UpdateObjectHierarchyRequest) returns (.UpdateObjectHierarchyResponse); +} + diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/StorageService.proto b/src/AVEVA.Historian.Client/Grpc/Protos/StorageService.proto new file mode 100644 index 0000000..352d149 --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/StorageService.proto @@ -0,0 +1,417 @@ +// Recovered from StorageService.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + +import "Status.proto"; + +option csharp_namespace = "ArchestrA.Grpc.Contract.Storage"; + +message GetInterfaceVersionRequest { +} + +message GetInterfaceVersionResponse { + uint32 uiError = 1; + uint32 uiVersion = 2; +} + +message OpenStorageConnectionRequest { + string HostName = 1; + string EnginePath = 2; + uint32 FreeDiskSpace = 3; + string ProcessName = 4; + uint32 ProcessId = 5; + string UserName = 6; + bytes Password = 7; + uint32 PwdLength = 8; + uint32 ClientType = 9; + uint32 ClientVersion = 10; + uint32 ConnectionMode = 11; + uint32 ConnectionTimeout = 12; + string StorageSessionId = 13; +} + +message OpenStorageConnectionResponse { + .Status status = 1; + string StorageSessionId = 2; + uint32 Handle = 3; + uint64 ConnectionTime = 4; + uint32 ServerStatus = 5; +} + +message CloseStorageConnectionRequest { + uint32 Handle = 1; +} + +message CloseStorageConnectionResponse { + .Status status = 1; +} + +message PingRequest { + uint32 Handle = 1; +} + +message PingResponse { + .Status status = 1; + uint32 OutByteCount = 2; + bytes OutBuff = 3; +} + +message AddTagsRequest { + uint32 Handle = 1; + uint32 ElementCount = 2; + uint32 InByteCount = 3; + bytes InBuff = 4; +} + +message AddTagsResponse { + .Status status = 1; + uint32 OutByteCount = 2; + bytes OutBuff = 3; +} + +message RegisterTagsRequest { + uint32 Handle = 1; + uint32 ElementCount = 2; + uint32 InByteCount = 3; + bytes InBuff = 4; +} + +message RegisterTagsResponse { + .Status status = 1; + uint32 OutByteCount = 2; + bytes OutBuff = 3; +} + +message AddStreamValuesRequest { + uint32 Handle = 1; + uint32 Size = 2; + bytes Buffer = 3; +} + +message AddStreamValuesResponse { + .Status status = 1; +} + +message GetTagIdsRequest { + uint32 Handle = 1; + uint32 Sequence = 2; +} + +message GetTagIdsResponse { + .Status status = 1; + uint32 Sequence = 2; + uint32 Size = 3; + bytes TagIds = 4; +} + +message GetTagsRequest { + uint32 Handle = 1; + uint32 TagIdsSize = 2; + bytes TagIds = 3; + uint32 Sequence = 4; +} + +message GetTagsResponse { + .Status status = 1; + uint32 Sequence = 2; + uint32 TagInfosSize = 3; + bytes TagInfos = 4; +} + +message FlushMetadataRequest { + uint32 Handle = 1; + uint32 TagIdsSize = 2; + bytes TagIds = 3; +} + +message FlushMetadataResponse { + .Status status = 1; +} + +message FlushDataRequest { + uint32 Handle = 1; +} + +message FlushDataResponse { + .Status status = 1; +} + +message LoadBlocksRequest { + uint32 Handle = 1; + uint32 Sequence = 2; +} + +message LoadBlocksResponse { + .Status status = 1; + uint32 Sequence = 2; + uint32 HistoryBlockSize = 3; + bytes HistoryBlocks = 4; +} + +message GetSnapshotsRequest { + uint32 Handle = 1; + uint64 BlockStartTime = 2; + uint32 Sequence = 3; +} + +message GetSnapshotsResponse { + .Status status = 1; + uint32 Sequence = 2; + uint32 SnapshotSize = 3; + bytes Snapshot = 4; +} + +message StartQuerySnapshotRequest { + uint32 Handle = 1; + uint64 BlockStartTime = 2; + uint32 SnapshotInfoSize = 3; + bytes SnapshotInfo = 4; + uint32 SnapshotQueryId = 5; +} + +message StartQuerySnapshotResponse { + .Status status = 1; + uint32 SnapshotQueryId = 2; +} + +message NextQuerySnapshotRequest { + uint32 Handle = 1; + uint32 SnapshotQueryId = 2; + uint32 Sequence = 3; +} + +message NextQuerySnapshotResponse { + .Status status = 1; + uint32 Sequence = 2; + uint32 SnapshotSize = 3; + bytes Snapshot = 4; +} + +message EndSnapshotRequest { + uint32 Handle = 1; + uint32 SnapshotQueryId = 2; + uint64 BlockStartTime = 3; + uint32 SnapshotInfoSize = 4; + bytes SnapshotInfo = 5; + bool IsDeleteSnapshot = 6; +} + +message EndSnapshotResponse { + .Status status = 1; +} + +message StopRequest { + uint32 Handle = 1; +} + +message StopResponse { + .Status status = 1; +} + +message ClearTagidPairsRequest { + uint32 Handle = 1; +} + +message ClearTagidPairsResponse { + .Status status = 1; +} + +message AddTagidPairsRequest { + uint32 Handle = 1; + uint32 ElementCount = 2; + uint32 InByteCount = 3; + bytes InBuff = 4; +} + +message AddTagidPairsResponse { + .Status status = 1; +} + +message GetSFParameterRequest { + uint32 Handle = 1; + string ParameterName = 2; +} + +message GetSFParameterResponse { + .Status status = 1; + string ParamaterValue = 2; +} + +message SetSFParameterRequest { + uint32 Handle = 1; + string ParamaterName = 2; + string ParamaterValue = 3; +} + +message SetSFParameterResponse { + .Status status = 1; +} + +message SendSnapshotBeginRequest { + uint32 Handle = 1; + uint64 TotalSize = 2; + uint64 StartTime = 3; + uint64 EndTime = 4; + string StorageSessionId = 5; +} + +message SendSnapshotBeginResponse { + .Status status = 1; + string StorageSessionId = 2; + uint32 QueryId = 3; +} + +message SendSnapshotEndRequest { + uint32 Handle = 1; + string StorageSessionId = 2; + uint32 QueryId = 3; + uint32 TimeRangeSize = 4; + bytes TimeRangeBytes = 5; +} + +message SendSnapshotEndResponse { + .Status status = 1; +} + +message SendSnapshotRequest { + uint32 Handle = 1; + string StorageSessionId = 2; + uint32 QueryId = 3; + uint32 Size = 4; + uint64 SnapShotChunkOffset = 5; + bytes Buffer = 6; +} + +message SendSnapshotResponse { + .Status status = 1; +} + +message DeleteSnapshotRequest { + uint32 Handle = 1; + uint64 StartTime = 2; + uint32 SnapshotInfoSize = 3; + bytes SnapshotInfo = 4; +} + +message DeleteSnapshotResponse { + .Status status = 1; +} + +message AddStreamValues2Request { + uint32 Handle = 1; + string ShardId = 2; + bytes Buffer = 3; +} + +message AddStreamValues2Response { + .Status status = 1; +} + +message ClearShardTagidsRequest { + uint32 Handle = 1; +} + +message ClearShardTagidsResponse { + .Status status = 1; +} + +message AddShardTagidsRequest { + uint32 Handle = 1; + bytes Buffer = 2; +} + +message AddShardTagidsResponse { + .Status status = 1; +} + +message SplitUnknownShardsRequest { + uint32 Handle = 1; +} + +message SplitUnknownShardsResponse { + .Status status = 1; +} + +message GetRemainingSnapshotsSizeRequest { + uint32 Handle = 1; +} + +message GetRemainingSnapshotsSizeResponse { + .Status status = 1; + uint64 SnapshotSize = 2; +} + +message DeleteTagsRequest { + uint32 Handle = 1; + bytes Buffer = 2; +} + +message DeleteTagsResponse { + .Status status = 1; +} + +message OpenStorageConnection2Request { + bytes InParameters = 1; +} + +message OpenStorageConnection2Response { + .Status status = 1; + bytes OutParmaters = 2; +} + +message ValidateClientCredentialRequest { + string Handle = 1; + bytes InBuff = 2; +} + +message ValidateClientCredentialResponse { + .Status status = 1; + bytes OutBuff = 2; +} + +message GetInfoRequest { + string Request = 1; +} + +message GetInfoResponse { + .Status status = 1; + bytes info = 2; +} + +service StorageService { + rpc GetInterfaceVersion (.GetInterfaceVersionRequest) returns (.GetInterfaceVersionResponse); + rpc OpenStorageConnection (.OpenStorageConnectionRequest) returns (.OpenStorageConnectionResponse); + rpc CloseStorageConnection (.CloseStorageConnectionRequest) returns (.CloseStorageConnectionResponse); + rpc Ping (.PingRequest) returns (.PingResponse); + rpc AddTags (.AddTagsRequest) returns (.AddTagsResponse); + rpc RegisterTags (.RegisterTagsRequest) returns (.RegisterTagsResponse); + rpc AddStreamValues (.AddStreamValuesRequest) returns (.AddStreamValuesResponse); + rpc GetTagIds (.GetTagIdsRequest) returns (.GetTagIdsResponse); + rpc GetTags (.GetTagsRequest) returns (.GetTagsResponse); + rpc FlushMetadata (.FlushMetadataRequest) returns (.FlushMetadataResponse); + rpc FlushData (.FlushDataRequest) returns (.FlushDataResponse); + rpc LoadBlocks (.LoadBlocksRequest) returns (.LoadBlocksResponse); + rpc GetSnapshots (.GetSnapshotsRequest) returns (.GetSnapshotsResponse); + rpc StartQuerySnapshot (.StartQuerySnapshotRequest) returns (.StartQuerySnapshotResponse); + rpc NextQuerySnapshot (.NextQuerySnapshotRequest) returns (.NextQuerySnapshotResponse); + rpc EndSnapshot (.EndSnapshotRequest) returns (.EndSnapshotResponse); + rpc Stop (.StopRequest) returns (.StopResponse); + rpc ClearTagidPairs (.ClearTagidPairsRequest) returns (.ClearTagidPairsResponse); + rpc AddTagidPairs (.AddTagidPairsRequest) returns (.AddTagidPairsResponse); + rpc GetSFParameter (.GetSFParameterRequest) returns (.GetSFParameterResponse); + rpc SetSFParameter (.SetSFParameterRequest) returns (.SetSFParameterResponse); + rpc SendSnapshotBegin (.SendSnapshotBeginRequest) returns (.SendSnapshotBeginResponse); + rpc SendSnapshotEnd (.SendSnapshotEndRequest) returns (.SendSnapshotEndResponse); + rpc SendSnapshot (.SendSnapshotRequest) returns (.SendSnapshotResponse); + rpc DeleteSnapshot (.DeleteSnapshotRequest) returns (.DeleteSnapshotResponse); + rpc AddStreamValues2 (.AddStreamValues2Request) returns (.AddStreamValues2Response); + rpc ClearShardTagids (.ClearShardTagidsRequest) returns (.ClearShardTagidsResponse); + rpc AddShardTagids (.AddShardTagidsRequest) returns (.AddShardTagidsResponse); + rpc SplitUnknownShards (.SplitUnknownShardsRequest) returns (.SplitUnknownShardsResponse); + rpc GetRemainingSnapshotsSize (.GetRemainingSnapshotsSizeRequest) returns (.GetRemainingSnapshotsSizeResponse); + rpc DeleteTags (.DeleteTagsRequest) returns (.DeleteTagsResponse); + rpc OpenStorageConnection2 (.OpenStorageConnection2Request) returns (.OpenStorageConnection2Response); + rpc ValidateClientCredential (.ValidateClientCredentialRequest) returns (.ValidateClientCredentialResponse); + rpc GetInfo (.GetInfoRequest) returns (.GetInfoResponse); +} + diff --git a/src/AVEVA.Historian.Client/Grpc/Protos/TransactionService.proto b/src/AVEVA.Historian.Client/Grpc/Protos/TransactionService.proto new file mode 100644 index 0000000..2c0e02c --- /dev/null +++ b/src/AVEVA.Historian.Client/Grpc/Protos/TransactionService.proto @@ -0,0 +1,92 @@ +// Recovered from TransactionService.proto (AVEVA Historian SDK 2023 R2, Archestra.Grpc.Contract). +// Reconstructed from the embedded protobuf FileDescriptor; field numbers are authoritative. +syntax = "proto3"; + +import "Status.proto"; + +option csharp_namespace = "ArchestrA.Grpc.Contract.Transaction"; + +message ForwardSnapshotRequest { + string strHandle = 1; + string strSessionID = 2; + uint32 queryID = 3; + uint64 snapShotChunkOffset = 4; + bytes btInput = 5; +} + +message ForwardSnapshotResponse { + .Status status = 1; +} + +message ForwardSnapshotBeginRequest { + string strHandle = 1; + uint64 totalSize = 2; + uint64 startTime = 3; + uint64 endTime = 4; +} + +message ForwardSnapshotBeginResponse { + string strSessionID = 1; + uint32 queryID = 2; + .Status status = 3; +} + +message ForwardSnapshotEndRequest { + string strHandle = 1; + string strSessionID = 2; + uint32 queryID = 3; + bytes timeRange = 4; +} + +message ForwardSnapshotEndResponse { + bytes tagIds = 1; + .Status status = 2; +} + +message GetTransactionInterfaceVersionRequest { +} + +message GetTransactionInterfaceVersionResponse { + uint32 error = 1; + uint32 version = 2; +} + +message AddNonStreamValuesBeginRequest { + string strHandle = 1; +} + +message AddNonStreamValuesBeginResponse { + .Status status = 1; + string strTransactionId = 2; +} + +message AddNonStreamValuesRequest { + string strHandle = 1; + string strTransactionId = 2; + bytes btInput = 3; +} + +message AddNonStreamValuesResponse { + .Status status = 1; +} + +message AddNonStreamValuesEndRequest { + string strHandle = 1; + string strTransactionId = 2; + bool bCommit = 3; +} + +message AddNonStreamValuesEndResponse { + .Status status = 1; +} + +service TransactionService { + rpc ForwardSnapshot (.ForwardSnapshotRequest) returns (.ForwardSnapshotResponse); + rpc ForwardSnapshotBegin (.ForwardSnapshotBeginRequest) returns (.ForwardSnapshotBeginResponse); + rpc ForwardSnapshotEnd (.ForwardSnapshotEndRequest) returns (.ForwardSnapshotEndResponse); + rpc GetTransactionInterfaceVersion (.GetTransactionInterfaceVersionRequest) returns (.GetTransactionInterfaceVersionResponse); + rpc AddNonStreamValuesBegin (.AddNonStreamValuesBeginRequest) returns (.AddNonStreamValuesBeginResponse); + rpc AddNonStreamValues (.AddNonStreamValuesRequest) returns (.AddNonStreamValuesResponse); + rpc AddNonStreamValuesEnd (.AddNonStreamValuesEndRequest) returns (.AddNonStreamValuesEndResponse); +} + diff --git a/src/AVEVA.Historian.Client/HistorianClientOptions.cs b/src/AVEVA.Historian.Client/HistorianClientOptions.cs index 60c6d06..afd7174 100644 --- a/src/AVEVA.Historian.Client/HistorianClientOptions.cs +++ b/src/AVEVA.Historian.Client/HistorianClientOptions.cs @@ -6,6 +6,9 @@ public sealed class HistorianClientOptions { public const int DefaultPort = 32568; + /// Default TCP port of the 2023 R2 Historian Client Access Point gRPC endpoint. + public const int DefaultGrpcPort = 32565; + public required string Host { get; init; } public int Port { get; init; } = DefaultPort; @@ -49,4 +52,14 @@ public sealed class HistorianClientOptions /// that don't validate a server certificate. /// public string? ServerDnsIdentity { get; init; } + + /// + /// For : when true the channel uses TLS + /// (https://); when false it uses plaintext (http://). Matches the stock + /// 2023 R2 client's securedConnection flag. The TLS host is taken from + /// when set (to match the server certificate's name), + /// otherwise . When is + /// true the server certificate chain is not validated. Default false. + /// + public bool GrpcUseTls { get; init; } } diff --git a/src/AVEVA.Historian.Client/HistorianTransport.cs b/src/AVEVA.Historian.Client/HistorianTransport.cs index 4044092..9d6e780 100644 --- a/src/AVEVA.Historian.Client/HistorianTransport.cs +++ b/src/AVEVA.Historian.Client/HistorianTransport.cs @@ -4,5 +4,12 @@ public enum HistorianTransport { LocalPipe = 0, RemoteTcpIntegrated = 1, - RemoteTcpCertificate = 2 + RemoteTcpCertificate = 2, + + /// + /// 2023 R2 gRPC transport (Historian Client Access Point gRPC-Web endpoint, default + /// TCP port 32565). Carries the same native binary payloads as the WCF transports inside + /// protobuf bytes fields. See Grpc/HistorianGrpcReadOrchestrator. + /// + RemoteGrpc = 3 } diff --git a/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs b/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs index 68bd587..8f9a2fd 100644 --- a/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs +++ b/src/AVEVA.Historian.Client/Protocol/Historian2020ProtocolDialect.cs @@ -1,3 +1,4 @@ +using AVEVA.Historian.Client.Grpc; using AVEVA.Historian.Client.Models; using AVEVA.Historian.Client.Wcf; @@ -12,23 +13,28 @@ internal sealed class Historian2020ProtocolDialect _options = options ?? throw new ArgumentNullException(nameof(options)); } + private bool UseGrpc => _options.Transport == HistorianTransport.RemoteGrpc; + public IAsyncEnumerable ReadRawAsync(string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken cancellationToken) { - HistorianWcfReadOrchestrator orchestrator = new(_options); - return orchestrator.ReadRawAsync(tag, startUtc, endUtc, maxValues, cancellationToken); + return UseGrpc + ? new HistorianGrpcReadOrchestrator(_options).ReadRawAsync(tag, startUtc, endUtc, maxValues, cancellationToken) + : new HistorianWcfReadOrchestrator(_options).ReadRawAsync(tag, startUtc, endUtc, maxValues, cancellationToken); } public IAsyncEnumerable ReadAggregateAsync(string tag, DateTime startUtc, DateTime endUtc, RetrievalMode mode, TimeSpan interval, CancellationToken cancellationToken) { - HistorianWcfReadOrchestrator orchestrator = new(_options); - return orchestrator.ReadAggregateAsync(tag, startUtc, endUtc, mode, interval, cancellationToken); + return UseGrpc + ? new HistorianGrpcReadOrchestrator(_options).ReadAggregateAsync(tag, startUtc, endUtc, mode, interval, cancellationToken) + : new HistorianWcfReadOrchestrator(_options).ReadAggregateAsync(tag, startUtc, endUtc, mode, interval, cancellationToken); } public Task> ReadAtTimeAsync(string tag, IReadOnlyList timestampsUtc, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - HistorianWcfReadOrchestrator orchestrator = new(_options); - return orchestrator.ReadAtTimeAsync(tag, timestampsUtc, cancellationToken); + return UseGrpc + ? new HistorianGrpcReadOrchestrator(_options).ReadAtTimeAsync(tag, timestampsUtc, cancellationToken) + : new HistorianWcfReadOrchestrator(_options).ReadAtTimeAsync(tag, timestampsUtc, cancellationToken); } public IAsyncEnumerable ReadBlocksAsync(string tag, DateTime startUtc, DateTime endUtc, CancellationToken cancellationToken) diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs new file mode 100644 index 0000000..e4760a1 --- /dev/null +++ b/src/AVEVA.Historian.Client/Wcf/HistorianNativeHandshake.cs @@ -0,0 +1,165 @@ +using System.Buffers.Binary; +using System.Diagnostics; + +namespace AVEVA.Historian.Client.Wcf; + +/// +/// Transport-agnostic pieces of the native Historian connect handshake: building the +/// OpenConnection3 v6 request buffer, running the SSPI/NTLM token-exchange rounds, and +/// decoding the OpenConnection response. Shared by the WCF/MDAS path +/// () and the 2023 R2 gRPC path +/// (Grpc.HistorianGrpcReadOrchestrator). The byte payloads are identical across +/// transports — only the envelope (WCF operation vs gRPC method) differs. +/// +internal static class HistorianNativeHandshake +{ + private const int CredentialBlockSizeBytes = 1026; + private const int OpenConnectionMinResponseLength = 5; + private const int MaxTokenRounds = 8; + private const string ClientNodeNameFallback = "AVEVA.Historian.Client"; + private const string ClientDataSourceId = "2020.406.2652.2"; + private const string ClientDllVersionString = "2020.406.2652.2"; + private const byte NativeClientType = 4; + private const byte NativeClientCommonInfoFormatVersion = 4; + private const ushort NativeHcalVersion = 17; + private const uint NativeClientVersionInt = 999_999; + private const ushort NativeOpen2ClientVersion = 9; + + /// Result of one transport-level credential-token exchange. + internal readonly record struct TokenExchangeResult(bool Success, byte[] ServerOutput, byte[] Error); + + /// + /// Performs a single credential-token round on the wire. is the + /// upper-case context-key GUID, is the AVEVA-wrapped SSPI + /// token (round byte + length + token). The WCF path maps this to + /// Hist.ValidateClientCredential; the gRPC path maps it to + /// HistoryService.ExchangeKey (the renamed handshake op). + /// + internal delegate TokenExchangeResult TokenExchange(string handle, byte[] wrappedToken, int round); + + /// + /// Drives the SSPI/NTLM negotiate loop against the supplied + /// delegate until the server signals terminal success. Mirrors the native two-round + /// (69→239, 93→1) sequence. + /// + public static void RunTokenRounds( + TokenExchange exchange, + Guid contextKey, + HistorianClientOptions options, + CancellationToken cancellationToken) + { + using HistorianSspiClient sspi = options.IntegratedSecurity + ? new HistorianSspiClient(options.TargetSpn) + : new HistorianSspiClient(options.TargetSpn, ParseDomain(options.UserName), ParseUserName(options.UserName), options.Password); + + string handle = contextKey.ToString("D").ToUpperInvariant(); + byte[] incoming = []; + + for (int round = 0; round < MaxTokenRounds; round++) + { + cancellationToken.ThrowIfCancellationRequested(); + + HistorianSspiStepResult step = sspi.Next(incoming); + byte[] outgoing = step.Token; + HistorianWcfAuthenticationProtocol.TryApplyNativeNtlmNegotiateVersionFlag(outgoing); + byte[] wrapped = HistorianWcfAuthenticationProtocol.WrapValidateClientCredentialToken(round == 0, outgoing); + + TokenExchangeResult result = exchange(handle, wrapped, round); + byte[] serverOutput = result.ServerOutput ?? []; + byte[] error = result.Error ?? []; + + if (!result.Success) + { + throw new InvalidOperationException($"Credential token round {round} rejected (errorLen={error.Length})."); + } + + ValidateClientCredentialResponse? response = HistorianWcfAuthenticationProtocol.TryReadValidateClientCredentialResponse(serverOutput); + if (response is null || !response.Continue) + { + return; + } + + incoming = response.Token; + if (step.IsCompleted && incoming.Length == 0) + { + return; + } + } + + throw new InvalidOperationException($"Credential token exchange exceeded {MaxTokenRounds} rounds without terminal success."); + } + + /// + /// Builds the native OpenConnection3 (Open2) version-6 request buffer. Identical bytes are + /// sent over WCF (Hist.OpenConnection2) and gRPC + /// (HistoryService.OpenConnection.btConnectionRequest). + /// + public static byte[] BuildOpenConnection3Request(string host, Guid contextKey, uint connectionMode) + { + Process current = Process.GetCurrentProcess(); + string machineName = Environment.MachineName; + string processName = string.IsNullOrEmpty(current.ProcessName) ? ClientNodeNameFallback : current.ProcessName; + _ = host; // host reserved for remote-orchestrator extension + + HistorianOpen2Request open2 = new( + HostName: machineName, + ProcessName: string.Empty, + ProcessId: checked((uint)current.Id), + UserName: string.Empty, + Password: [], + ClientType: NativeClientType, + ClientVersion: NativeOpen2ClientVersion, + ConnectionMode: connectionMode, + MetadataNamespace: HistorianMetadataNamespace.Empty); + + HistorianClientCommonInfo commonInfo = new( + FormatVersion: NativeClientCommonInfoFormatVersion, + ServerNodeName: machineName, + ClientNodeName: processName, + ProcessId: checked((uint)current.Id), + HcalVersion: NativeHcalVersion, + ProcessName: string.Empty, + Proxy: string.Empty, + DataSourceId: ClientDataSourceId, + ShardId: Guid.Empty, + ClientVersion: NativeClientVersionInt, + ClientTimestamp: (ulong)DateTime.UtcNow.ToFileTimeUtc(), + ClientDllVersion: ClientDllVersionString); + + return HistorianOpen2Protocol.SerializeNativeOpenConnection3Version6( + open2, + commonInfo, + contextKey, + credentialBlock: new byte[CredentialBlockSizeBytes]); + } + + /// + /// Decodes the OpenConnection response blob: byte 0 = protocol version, bytes 1..4 = + /// transient /Retr client handle (UInt32 LE), bytes 5..20 = storage session GUID. + /// + public static (uint ClientHandle, Guid StorageSessionId) ParseOpenConnectionResponse(ReadOnlySpan response) + { + if (response.Length < OpenConnectionMinResponseLength) + { + throw new InvalidOperationException($"OpenConnection response too short (ResponseLen={response.Length})."); + } + + uint clientHandle = BinaryPrimitives.ReadUInt32LittleEndian(response.Slice(1, 4)); + Guid storageSessionId = response.Length >= 21 ? new Guid(response.Slice(5, 16)) : Guid.Empty; + return (clientHandle, storageSessionId); + } + + private static string ParseDomain(string userName) + { + if (string.IsNullOrEmpty(userName)) return string.Empty; + int slash = userName.IndexOf('\\'); + return slash > 0 ? userName[..slash] : string.Empty; + } + + private static string ParseUserName(string userName) + { + if (string.IsNullOrEmpty(userName)) return string.Empty; + int slash = userName.IndexOf('\\'); + return slash > 0 ? userName[(slash + 1)..] : userName; + } +} diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs index eaa3396..ed0a52e 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs @@ -1,6 +1,3 @@ -using System.Buffers.Binary; -using System.Diagnostics; -using System.Runtime.Versioning; using System.ServiceModel; using System.ServiceModel.Channels; using AVEVA.Historian.Client.Wcf.Contracts; @@ -10,12 +7,6 @@ namespace AVEVA.Historian.Client.Wcf; internal static class HistorianWcfAuthChainHelper { private const int OpenConnection3MinResponseLength = 5; - private const int CredentialBlockSizeBytes = 1026; - private const int MaxValClRounds = 8; - private const string ClientNodeNameFallback = "AVEVA.Historian.Client"; - private const string ClientDataSourceId = "2020.406.2652.2"; - private const string ClientDllVersionString = "2020.406.2652.2"; - private const byte NativeClientType = 4; public const uint NativeIntegratedReadOnlyConnectionMode = 0x402; public const uint NativeIntegratedEventConnectionMode = 0x501; /// @@ -25,10 +16,6 @@ internal static class HistorianWcfAuthChainHelper /// Open2 is opened with 0x402 (read-only); 0x401 unlocks write capability. /// public const uint NativeIntegratedWriteEnabledConnectionMode = 0x401; - private const byte NativeClientCommonInfoFormatVersion = 4; - private const ushort NativeHcalVersion = 17; - private const uint NativeClientVersionInt = 999_999; - private const ushort NativeOpen2ClientVersion = 9; /// /// Runs Hist.GetV → Hist.ValCl × N → Hist.Open2 against the configured /Hist endpoint and @@ -61,7 +48,7 @@ internal static class HistorianWcfAuthChainHelper historyChannel.GetInterfaceVersion(out _); RunValClRounds(historyChannel, contextKey, options, cancellationToken); - byte[] open2Request = BuildOpenConnection3Request(options.Host, contextKey, connectionMode); + byte[] open2Request = HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); bool open2Success = historyChannel.OpenConnection2(ref open2Request, out byte[] open2Response, out byte[] open2Error); open2Response ??= []; open2Error ??= []; @@ -71,10 +58,7 @@ internal static class HistorianWcfAuthChainHelper $"Open2 failed (Success={open2Success}, ResponseLen={open2Response.Length}, ErrorLen={open2Error.Length})."); } - uint clientHandle = BinaryPrimitives.ReadUInt32LittleEndian(open2Response.AsSpan(1, 4)); - Guid storageSessionId = open2Response.Length >= 21 - ? new Guid(open2Response.AsSpan(5, 16)) - : Guid.Empty; + (uint clientHandle, Guid storageSessionId) = HistorianNativeHandshake.ParseOpenConnectionResponse(open2Response); if (additionalSetup is not null) { @@ -98,97 +82,15 @@ internal static class HistorianWcfAuthChainHelper private static void RunValClRounds(IHistoryServiceContract2 channel, Guid contextKey, HistorianClientOptions options, CancellationToken cancellationToken) { - using HistorianSspiClient sspi = options.IntegratedSecurity - ? new HistorianSspiClient(options.TargetSpn) - : new HistorianSspiClient(options.TargetSpn, ParseDomain(options.UserName), ParseUserName(options.UserName), options.Password); - string handle = contextKey.ToString("D").ToUpperInvariant(); - byte[] incoming = []; - - for (int round = 0; round < MaxValClRounds; round++) - { - cancellationToken.ThrowIfCancellationRequested(); - - HistorianSspiStepResult step = sspi.Next(incoming); - byte[] outgoing = step.Token; - HistorianWcfAuthenticationProtocol.TryApplyNativeNtlmNegotiateVersionFlag(outgoing); - byte[] wrapped = HistorianWcfAuthenticationProtocol.WrapValidateClientCredentialToken(round == 0, outgoing); - - bool serverSuccess = channel.ValidateClientCredential(handle, wrapped, out byte[] serverOutput, out byte[] errorBuffer); - serverOutput ??= []; - errorBuffer ??= []; - - if (!serverSuccess) + HistorianNativeHandshake.RunTokenRounds( + (handle, wrapped, _) => { - throw new InvalidOperationException($"ValCl round {round} rejected (errorLen={errorBuffer.Length})."); - } - - ValidateClientCredentialResponse? response = HistorianWcfAuthenticationProtocol.TryReadValidateClientCredentialResponse(serverOutput); - if (response is null || !response.Continue) - { - return; - } - - incoming = response.Token; - if (step.IsCompleted && incoming.Length == 0) - { - return; - } - } - - throw new InvalidOperationException($"ValCl exceeded {MaxValClRounds} rounds without terminal success."); - } - - private static string ParseDomain(string userName) - { - if (string.IsNullOrEmpty(userName)) return string.Empty; - int slash = userName.IndexOf('\\'); - return slash > 0 ? userName[..slash] : string.Empty; - } - - private static string ParseUserName(string userName) - { - if (string.IsNullOrEmpty(userName)) return string.Empty; - int slash = userName.IndexOf('\\'); - return slash > 0 ? userName[(slash + 1)..] : userName; - } - - private static byte[] BuildOpenConnection3Request(string host, Guid contextKey, uint connectionMode) - { - Process current = Process.GetCurrentProcess(); - string machineName = Environment.MachineName; - string processName = string.IsNullOrEmpty(current.ProcessName) ? ClientNodeNameFallback : current.ProcessName; - _ = host; // host reserved for remote-orchestrator extension - - HistorianOpen2Request open2 = new( - HostName: machineName, - ProcessName: string.Empty, - ProcessId: checked((uint)current.Id), - UserName: string.Empty, - Password: [], - ClientType: NativeClientType, - ClientVersion: NativeOpen2ClientVersion, - ConnectionMode: connectionMode, - MetadataNamespace: HistorianMetadataNamespace.Empty); - - HistorianClientCommonInfo commonInfo = new( - FormatVersion: NativeClientCommonInfoFormatVersion, - ServerNodeName: machineName, - ClientNodeName: processName, - ProcessId: checked((uint)current.Id), - HcalVersion: NativeHcalVersion, - ProcessName: string.Empty, - Proxy: string.Empty, - DataSourceId: ClientDataSourceId, - ShardId: Guid.Empty, - ClientVersion: NativeClientVersionInt, - ClientTimestamp: (ulong)DateTime.UtcNow.ToFileTimeUtc(), - ClientDllVersion: ClientDllVersionString); - - return HistorianOpen2Protocol.SerializeNativeOpenConnection3Version6( - open2, - commonInfo, + bool serverSuccess = channel.ValidateClientCredential(handle, wrapped, out byte[] serverOutput, out byte[] errorBuffer); + return new HistorianNativeHandshake.TokenExchangeResult(serverSuccess, serverOutput ?? [], errorBuffer ?? []); + }, contextKey, - credentialBlock: new byte[CredentialBlockSizeBytes]); + options, + cancellationToken); } private static void CloseChannelSafely(ICommunicationObject channel) diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs index d84da8e..7cb44be 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs @@ -371,7 +371,7 @@ internal sealed class HistorianWcfReadOrchestrator } } - private static HistorianDataQueryRequest BuildDataQueryRequest(string tag, DateTime startUtc, DateTime endUtc, int maxValues) + internal static HistorianDataQueryRequest BuildDataQueryRequest(string tag, DateTime startUtc, DateTime endUtc, int maxValues) { return new HistorianDataQueryRequest( TagNames: [tag], @@ -382,7 +382,7 @@ internal sealed class HistorianWcfReadOrchestrator Option: string.Empty); } - private static HistorianDataQueryRequest BuildAggregateQueryRequest( + internal static HistorianDataQueryRequest BuildAggregateQueryRequest( string tag, DateTime startUtc, DateTime endUtc, @@ -427,7 +427,7 @@ internal sealed class HistorianWcfReadOrchestrator return (uint)mode; } - private static uint MapRetrievalModeToAggregationType(Models.RetrievalMode mode) => mode switch + internal static uint MapRetrievalModeToAggregationType(Models.RetrievalMode mode) => mode switch { Models.RetrievalMode.TimeWeightedAverage => 0, Models.RetrievalMode.Interpolated => 3, diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs new file mode 100644 index 0000000..dcd71d6 --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs @@ -0,0 +1,63 @@ +using AVEVA.Historian.Client.Models; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Live integration tests for the 2023 R2 RemoteGrpc transport. Gated on a dedicated +/// HISTORIAN_GRPC_HOST env var (plus HISTORIAN_TEST_TAG) so they skip cleanly until +/// a 2023 R2 Historian is available. Optional: +/// HISTORIAN_GRPC_PORT (default 32565), HISTORIAN_GRPC_TLS (true/false), +/// HISTORIAN_USER / HISTORIAN_PASSWORD (explicit creds; otherwise IntegratedSecurity), +/// HISTORIAN_GRPC_DNSID (server certificate name when connecting by IP over TLS). +/// +public sealed class HistorianGrpcIntegrationTests +{ + [Fact] + public async Task ReadRawAsync_OverGrpc_ReturnsAtLeastOneRow() + { + string? host = Environment.GetEnvironmentVariable("HISTORIAN_GRPC_HOST"); + string? testTag = Environment.GetEnvironmentVariable("HISTORIAN_TEST_TAG"); + if (string.IsNullOrWhiteSpace(host) || string.IsNullOrWhiteSpace(testTag)) + { + return; + } + + HistorianClient client = new(BuildOptions(host)); + + DateTime endUtc = DateTime.UtcNow; + DateTime startUtc = endUtc - TimeSpan.FromDays(7); + + List samples = []; + await foreach (HistorianSample sample in client.ReadRawAsync(testTag, startUtc, endUtc, maxValues: 8, CancellationToken.None)) + { + samples.Add(sample); + } + + Assert.NotEmpty(samples); + Assert.All(samples, s => Assert.Equal(testTag, s.TagName)); + } + + private static HistorianClientOptions BuildOptions(string host) + { + string? user = Environment.GetEnvironmentVariable("HISTORIAN_USER"); + string? password = Environment.GetEnvironmentVariable("HISTORIAN_PASSWORD"); + bool explicitCreds = !string.IsNullOrEmpty(user); + int port = int.TryParse(Environment.GetEnvironmentVariable("HISTORIAN_GRPC_PORT"), out int parsed) + ? parsed + : HistorianClientOptions.DefaultGrpcPort; + bool tls = string.Equals(Environment.GetEnvironmentVariable("HISTORIAN_GRPC_TLS"), "true", StringComparison.OrdinalIgnoreCase); + + return new HistorianClientOptions + { + Host = host, + Port = port, + Transport = HistorianTransport.RemoteGrpc, + GrpcUseTls = tls, + AllowUntrustedServerCertificate = tls, + ServerDnsIdentity = Environment.GetEnvironmentVariable("HISTORIAN_GRPC_DNSID"), + IntegratedSecurity = !explicitCreds, + UserName = user ?? string.Empty, + Password = password ?? string.Empty + }; + } +} diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcTransportTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcTransportTests.cs new file mode 100644 index 0000000..a67de94 --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcTransportTests.cs @@ -0,0 +1,114 @@ +using AVEVA.Historian.Client.Grpc; +using AVEVA.Historian.Client.Models; +using AVEVA.Historian.Client.Wcf; +using Google.Protobuf; +using ArchestrA.Grpc.Contract.Retrieval; +using GrpcHistory = ArchestrA.Grpc.Contract.History; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Unit coverage for the 2023 R2 RemoteGrpc transport — the parts that do not require a live +/// server: channel address/port resolution, metadata, transport routing, and the invariant that +/// gRPC request messages carry the same native byte buffers the WCF path uses. +/// +public sealed class HistorianGrpcTransportTests +{ + private static HistorianClientOptions Options( + string host = "histserver", + int port = HistorianClientOptions.DefaultPort, + bool tls = false, + string? dnsIdentity = null, + bool compression = false) => new() + { + Host = host, + Port = port, + Transport = HistorianTransport.RemoteGrpc, + GrpcUseTls = tls, + ServerDnsIdentity = dnsIdentity, + Compression = compression, + IntegratedSecurity = true + }; + + [Fact] + public void ResolvePort_DefaultWcfPort_SubstitutesGrpcDefault() + { + Assert.Equal(HistorianClientOptions.DefaultGrpcPort, HistorianGrpcChannelFactory.ResolvePort(Options(port: HistorianClientOptions.DefaultPort))); + } + + [Fact] + public void ResolvePort_ExplicitPort_IsHonoured() + { + Assert.Equal(443, HistorianGrpcChannelFactory.ResolvePort(Options(port: 443))); + } + + [Fact] + public void ResolveAddress_Plaintext_UsesHttpAndHost() + { + Assert.Equal("http://histserver:32565", HistorianGrpcChannelFactory.ResolveAddress(Options())); + } + + [Fact] + public void ResolveAddress_Tls_UsesHttpsAndHostWhenNoDnsIdentity() + { + Assert.Equal("https://histserver:32565", HistorianGrpcChannelFactory.ResolveAddress(Options(tls: true))); + } + + [Fact] + public void ResolveAddress_Tls_PrefersDnsIdentityForCertMatch() + { + string address = HistorianGrpcChannelFactory.ResolveAddress(Options(host: "10.0.0.5", tls: true, dnsIdentity: "localhost")); + Assert.Equal("https://localhost:32565", address); + } + + [Fact] + public void Create_CompressionDisabled_EmitsNoEncodingHeader() + { + using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(Options(compression: false)); + Assert.DoesNotContain(connection.Metadata, e => e.Key == "grpc-internal-encoding-request"); + } + + [Fact] + public void Create_CompressionEnabled_AdvertisesGzipRequestEncoding() + { + using HistorianGrpcConnection connection = HistorianGrpcChannelFactory.Create(Options(compression: true)); + global::Grpc.Core.Metadata.Entry entry = Assert.Single(connection.Metadata, e => e.Key == "grpc-internal-encoding-request"); + Assert.Equal("gzip", entry.Value); + } + + [Fact] + public void StartQueryRequest_CarriesNativeDataQueryBufferUnchanged() + { + // The gRPC envelope must wrap the exact bytes the WCF StartQuery2 path sends, so the + // already-reverse-engineered DataQueryRequest serializer is reused verbatim. + HistorianDataQueryRequest request = HistorianWcfReadOrchestrator.BuildDataQueryRequest( + "Tag.Counter", new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc), new DateTime(2026, 1, 2, 0, 0, 0, DateTimeKind.Utc), 100); + byte[] nativeBuffer = HistorianDataQueryProtocol.SerializeFullHistoryRequest(request); + + var message = new StartQueryRequest + { + UiHandle = 7, + UiQueryRequestType = HistorianDataQueryProtocol.QueryRequestTypeData, + BtRequestBuffer = ByteString.CopyFrom(nativeBuffer) + }; + + // Round-trip through protobuf and confirm the native buffer survives byte-for-byte. + byte[] wire = message.ToByteArray(); + var decoded = StartQueryRequest.Parser.ParseFrom(wire); + Assert.Equal(nativeBuffer, decoded.BtRequestBuffer.ToByteArray()); + Assert.Equal(7u, decoded.UiHandle); + Assert.Equal((uint)HistorianDataQueryProtocol.QueryRequestTypeData, decoded.UiQueryRequestType); + } + + [Fact] + public void OpenConnectionRequest_CarriesNativeOpen2BufferUnchanged() + { + byte[] open2 = HistorianNativeHandshake.BuildOpenConnection3Request( + "histserver", Guid.NewGuid(), HistorianWcfAuthChainHelper.NativeIntegratedReadOnlyConnectionMode); + + var message = new GrpcHistory.OpenConnectionRequest { BtConnectionRequest = ByteString.CopyFrom(open2) }; + var decoded = GrpcHistory.OpenConnectionRequest.Parser.ParseFrom(message.ToByteArray()); + + Assert.Equal(open2, decoded.BtConnectionRequest.ToByteArray()); + } +} From a530ae0f1097626a95012b46076c5a0211e979a1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 14:28:34 -0400 Subject: [PATCH 2/6] docs/plans: import 2023 R2 gRPC analysis + HCAL reimpl roadmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Version-control the planning docs alongside the code they describe: - grpc-transport.md — 2023 R2 gRPC transport analysis (sanitized source path) - hcal-capability-matrix.md — HistorianAccess surface x gRPC ops x histsdk status x feasibility tiers - hcal-roadmap.md — ordered build plan M0-M4 + cross-cutting workstreams - histevents.md — how a HistorianEvent reaches the DB (client->wire->server) Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/plans/grpc-transport.md | 265 ++++++++++++++++++++++++ docs/plans/hcal-capability-matrix.md | 166 +++++++++++++++ docs/plans/hcal-roadmap.md | 171 +++++++++++++++ docs/plans/histevents.md | 299 +++++++++++++++++++++++++++ 4 files changed, 901 insertions(+) create mode 100644 docs/plans/grpc-transport.md create mode 100644 docs/plans/hcal-capability-matrix.md create mode 100644 docs/plans/hcal-roadmap.md create mode 100644 docs/plans/histevents.md diff --git a/docs/plans/grpc-transport.md b/docs/plans/grpc-transport.md new file mode 100644 index 0000000..b81bcbc --- /dev/null +++ b/docs/plans/grpc-transport.md @@ -0,0 +1,265 @@ +# AVEVA Historian SDK 2023 R2 — gRPC Transport Analysis + +**Scope:** Documents the new gRPC transport that 2023 R2 adds to the Historian +Client Access Layer (HCAL). Kept deliberately **separate** from the main +`histsdk` reverse-engineering docs — this is 2023 R2 evidence, not the 2020/WCF +protocol the production SDK currently targets. + +**Source:** the 2023 R2 `HistorianSDK` installer (**not installed**). +`SDKSetup.msi` was laid out with `msiexec /a` (administrative extract, no +registration) into a local `msi-extract` staging dir, then the managed +assemblies were decompiled with `ilspycmd`. + +**Assembly versions analysed:** `2023.1219.4004.5` +(`Archestra.Grpc.Contract.dll`, `Archestra.Historian.GrpcClient.dll`, +`aahClientManaged.dll`). + +--- + +## 1. Headline finding + +The 2023 R2 gRPC transport is a **transport swap, not a protocol redesign.** +Every gRPC request/response wraps the **same opaque native binary buffers** that +the 2020/WCF-MDAS path already carries — `OpenConnection3` v6 buffer, NTLM/SSPI +`ValCl` tokens, `DataQueryRequest`, `GetNextQueryResultBuffer` row buffers, the +`Status` err blob, etc. — inside protobuf `bytes` fields. + +Concrete proof, from `Archestra.Historian.GrpcClient`: + +```csharp +// History/OpenConnection — same byte[] openParameters the WCF path built +OpenConnectionRequest request = new OpenConnectionRequest { + BtConnectionRequest = ByteString.CopyFrom(openParameters) +}; +// Retrieval/StartQuery — same queryType + DataQueryRequest bytes + handle +StartQueryRequest request = new StartQueryRequest { + UiHandle = handle, + UiQueryRequestType = queryRequestType, + BtRequestBuffer = ByteString.CopyFrom(requestBuffer) +}; +``` + +**Implication for the `histsdk` project:** all of the hard-won payload +serializers (`HistorianOpen2Protocol`, `HistorianDataQueryProtocol`, +`HistorianEventRowProtocol`, the SSPI `ValCl` token framing, the EnsT2 +`CTagMetadata` layout) transfer **unchanged**. Only the envelope around them +changes: protobuf-over-gRPC instead of binary-SOAP-over-`application/x-mdas`. +The WCF `[MessageParameter(Name=…)]` guessing that dominated the 2020 work is +gone — field names and numbers are explicit in the protobuf contract. + +--- + +## 2. Transport stack + +From `GrpcClientBase.InitializeBase(target, portNumber, securedConnection, certificateName, trusted)`: + +| Aspect | Value / behaviour | +|---|---| +| Library | `Grpc.Net.Client` + **`Grpc.Net.Client.Web`** (`GrpcWebHandler`) | +| Mode | **gRPC-Web**, `GrpcWebMode.GrpcWeb` (binary `application/grpc-web`, **not** `-text`) | +| HTTP version | **HTTP/1.1** (`GrpcWebHandler.HttpVersion = new Version(1,1)`) — *not* HTTP/2 | +| Address | `http://{target}:{port}` insecure, or `https://{certificateName}:{port}` secure | +| Inner handler | `HttpClientHandler` with custom `ServerCertificateCustomValidationCallback` | +| Compression | gzip on by default; request header `grpc-internal-encoding-request: gzip`; custom `CustomCompressionProvider` / `CustomGZipStream` used for bandwidth accounting | +| Default timeout | 60 s per call (`m_timeoutInSeconds = 60`, sent as gRPC deadline) | +| Interceptor | `ClientInterceptor` (logging hook, currently a no-op `LogCall`) | + +Because it is **gRPC-Web over HTTP/1.1**, the transport is proxy/firewall +friendly and does not require HTTP/2 negotiation — note `HistorianConnectionArgs.ProxyServer` +(e.g. `http://host:9480`) in the public API. + +### Port + +- **Default port `32565`** — `HistorianConnectionArgs.TcpPort`, *"the TCP port + of the Historian Client Access Point."* (Note this differs from the 2020 WCF + port `32568` the production SDK uses.) +- All services reach the **same host:port**; gRPC multiplexes by service path + (`/HistoryService/OpenConnection`, `/RetrievalService/StartQuery`, …). + +### Channel topology + +Five service stubs grouped into four wrapper clients, each constructing its own +`GrpcChannel` to the same endpoint: + +| Wrapper (`GrpcClientBase` subclass) | gRPC service stub(s) | +|---|---| +| `GrpcHistoryClient` | `HistoryService` + `TransactionService` (one channel) | +| `GrpcRetrievalClient` | `RetrievalService` | +| `GrpcStatusClient` | `StatusService` | +| `GrpcStorageClient` | `StorageService` | + +--- + +## 3. Authentication model (unchanged in substance) + +Auth is **still the native session handshake**, carried over gRPC instead of +WCF. There is **no per-call bearer/auth token in gRPC metadata** — the only +metadata sent is the gzip-encoding hint. Methods pass `m_metadata` (gzip) or +`null`; neither carries credentials. The server keys the session off the +`handle` GUID established by the handshake, exactly as the 2020 path does. + +Handshake operations (same byte payloads as 2020): + +- `HistoryService.GetInterfaceVersion` → version probe. +- `StorageService.ValidateClientCredential { string Handle; bytes InBuff }` + → `{ Status; bytes OutBuff }`. **`InBuff`/`OutBuff` carry the NTLM/SSPI + tokens** — same multi-round Negotiate exchange, same field names the 2020 + `ildasm` revealed (`inBuff`/`outBuff`), now first-class protobuf fields. +- `HistoryService.ExchangeKey { string StrHandle; bytes BtInput }` + → `{ Status; bytes BtOutput }` (key-exchange / cert path). +- `HistoryService.OpenConnection { bytes BtConnectionRequest }` + → `{ Status; bytes BtConnectionResponse }` — same `OpenConnection3` v6 + request buffer in, same 42-byte session blob out. + +Public-API security knobs (`aahClientManaged.xml`): + +- `HistorianConnectionArgs.ConnectionMode` — *"whether GRPC connection to the + Historian Server. **Default is true** (GRPC)."* This is the master switch + selecting gRPC vs legacy. +- `HistorianSecurityMode`: `None`, `Disabled`, `TransportWindows` + (Windows creds), `TransportCertificate` (server cert). +- `AllowUnTrustedConnection` → maps to the `trusted` arg; when false the client + bypasses X509 chain validation (`ValidateServerCertificate` returns true + early). Equivalent to the production SDK's `AllowUntrustedServerCertificate`. +- `AuthenticationMode` default `HistorianNative`; `CertificateInfo.CertificateName` + supplies the `https://{certificateName}:{port}` SNI/host identity. + +--- + +## 4. gRPC service surface (full RPC list) + +All methods are unary (`MethodType 0`). Names map 1:1 onto the 2020 WCF +operations the production SDK already understands. + +### HistoryService (`/HistoryService/…`) +`GetInterfaceVersion`, `ExchangeKey`, `OpenConnection`, `CloseConnection`, +`UpdateClientStatus`, `RegisterTags`, `EnsureTags`, `AddStreamValues`, +`AddTagExtendedPropertyGroups`, `AddTagExtendedProperties`, `StartJob`, +`GetJobStatus`, `DeleteTagExtendedProperties`, `DeleteTags`, +`AddTagLocalizedProperties`, `DeleteTagLocalizedProperties` + +### RetrievalService (`/RetrievalService/…`) +`GetRetrievalInterfaceVersion`, `StartQuery`, `GetNextQueryResultBuffer`, +`EndQuery`, `GetShardTagidsByTagnameAndSource`, `GetTagInfosFromName`, +`GetTagExtendedPropertiesFromName`, `ExecuteSqlCommand`, `StartEventQuery`, +`GetNextEventQueryResultBuffer`, `EndEventQuery`, `StartTagQuery`, `QueryTag`, +`EndTagQuery`, `GetTagLocalizedPropertiesFromName` + +### StatusService (`/StatusService/…`) +`GetStatusInterfaceVersion`, `GetSystemParameter`, `SendInfo`, `RequestInfo`, +`DeleteInfo`, `GetHistorianInfo`, `StartProcess`, `StopProcess`, `PingServer`, +`PingPipe`, `ConfigureAutoStartProcess`, `GetHistorianConsoleStatus`, +`GetRuntimeParameter`, `GetSystemTimeZoneName`, `SetHistorianConsoleStatus`, +`CanUpdateAreaHierarchy`, `UpdateAreaHierarchy`, `UpdateObjectHierarchy` + +### StorageService (`/StorageService/…`) +`GetInterfaceVersion`, `OpenStorageConnection`, `OpenStorageConnection2`, +`CloseStorageConnection`, `Ping`, `AddTags`, `RegisterTags`, `AddStreamValues`, +`AddStreamValues2`, `GetTagIds`, `GetTags`, `FlushMetadata`, `FlushData`, +`LoadBlocks`, `GetSnapshots`, `StartQuerySnapshot`, `NextQuerySnapshot`, +`EndSnapshot`, `Stop`, `ClearTagidPairs`, `AddTagidPairs`, `GetSFParameter`, +`SetSFParameter`, `SendSnapshotBegin`, `SendSnapshotEnd`, `SendSnapshot`, +`DeleteSnapshot`, `ClearShardTagids`, `AddShardTagids`, `SplitUnknownShards`, +`GetRemainingSnapshotsSize`, `DeleteTags`, `OpenStorageConnection2`, +`ValidateClientCredential`, `GetInfo` + +### TransactionService (`/TransactionService/…`) +`ForwardSnapshot`, `ForwardSnapshotBegin`, `ForwardSnapshotEnd`, +`GetTransactionInterfaceVersion`, `AddNonStreamValuesBegin`, +`AddNonStreamValues`, `AddNonStreamValuesEnd` + +> A separate `ArchestrA.CloudHistorian.Contract` assembly defines a parallel +> cloud-ingest contract (`AddHistorianValues`, `CreateTags`, `EnqueueTagDataPacket`, +> `Enqueue…`, etc.) used by `aahCloudConfigurator` / `online.wonderware.com`. +> Out of scope here; noted for completeness. + +--- + +## 5. Representative message shapes (protobuf field numbers) + +The universal result wrapper and the auth/query messages — note how thin they +are; the real structure lives inside the `bytes` fields. + +```proto +// Common result wrapper (ArchestrA.Grpc.Contract.RequestStatus) +message Status { + bool bSuccess = 1; // success flag (replaces WCF return-bool) + bytes btError = 2; // native error buffer (same type/code blob as WCF err) +} + +// HistoryService +message OpenConnectionRequest { bytes btConnectionRequest = 1; } // OpenConnection3 v6 buffer +message OpenConnectionResponse { Status status = 1; bytes btConnectionResponse = 2; } // 42-byte session blob +message ExchangeKeyRequest { string strHandle = 1; bytes btInput = 2; } + +// StorageService — Negotiate/NTLM handshake +message ValidateClientCredentialRequest { string handle = 1; bytes inBuff = 2; } +message ValidateClientCredentialResponse { Status status = 1; bytes outBuff = 2; } + +// RetrievalService +message StartQueryRequest { + uint32 uiHandle = 1; + uint32 uiQueryRequestType = 2; // RetrievalMode → QueryType, same mapping as 2020 + bytes btRequestBuffer = 3; // DataQueryRequest bytes, byte-identical to WCF +} +``` + +Across the contract the recurring pattern is `{ Status status; bytes }` +for responses and `{ [string handle][uint …] bytes }` for requests. + +**The full canonical IDL has been recovered.** All six `.proto` files were +rendered from the embedded `FileDescriptor`s and protoc-validated — see +`../out/proto/*.proto`, the portable `../out/archestra_grpc.fileset.pb` +(`FileDescriptorSet`), and the per-message field dump `../out/grpc-contract-dump.md`. +`../out/README.md` explains the contract quirks (global proto package, cross-file +name collisions, all-unary RPCs) and the 2020→gRPC read-path mapping. Regenerate +with the `protodump/` tool. + +--- + +## 6. What this means for histsdk (if a gRPC transport is ever added) + +This is **not** a request to implement anything — recording the path: + +1. Add a transport enum value (e.g. `RemoteGrpc`) alongside `LocalPipe` / + `RemoteTcpIntegrated` / `RemoteTcpCertificate`. +2. Reference `Grpc.Net.Client` + `Grpc.Net.Client.Web`; build a + `GrpcChannel.ForAddress("http(s)://host:32565", { HttpHandler = + GrpcWebHandler(GrpcWeb, HttpClientHandler), … })` with HTTP/1.1. +3. Reuse **every existing payload serializer unchanged** — feed the same byte + buffers into the protobuf `bytes` fields instead of MDAS bodies. The + orchestrator call order (`GetV → ValCl×N → Open2 → Retr.GetV → + IsOriginalAllowed → StartQuery → GetNextQueryResultBuffer…`) is identical. +4. Auth: still the SSPI/Negotiate token loop via `ValidateClientCredential`, + carried in `inBuff`/`outBuff`. No per-call gRPC auth metadata needed. +5. Biggest win: **no WCF `[MessageParameter]` reverse-engineering** — the + protobuf field numbers are authoritative and stable. + +Caveat: this is the **2023 R2** server contract. The production SDK targets a +2020-era server; whether that server exposes the gRPC HCAP endpoint at all is a +server-version question, not a client one. Treat this as forward-looking. + +--- + +## 7. Artifacts (all under the separate analysis folder, none committed to histsdk) + +``` +histsdk-2023r2-analysis/ + msi-extract/ # msiexec /a layout of SDKSetup.msi + bin/ # copied key assemblies + aahClientManaged.xml + decompiled/ + Archestra.Grpc.Contract/ # full protobuf contract (services + messages) + Archestra.Historian.GrpcClient/ # transport wrappers (channel/auth/calls) + ArchestrA.CloudHistorian.Contract/ # cloud-ingest contract (out of scope) + protodump/ # .NET 10 tool: descriptor graph -> .proto / dump + out/ + proto/*.proto # recovered, protoc-validated IDL (6 files) + archestra_grpc.fileset.pb # portable FileDescriptorSet (grpcurl/buf/protoc) + grpc-contract-dump.md # per-message field dump + service tables + README.md # artifact guide + contract quirks + read-path map + docs/grpc-transport.md # this file +``` + +gRPC redist proof in the installer: +`Redist/HistorianSDK 2023 R2/x64/{GRPCCore,GRPCNetClient,HistorianGRPCClient,HistorianGRPCContract,Protobuf}.msm` +plus shipped `Grpc.Net.Client*.dll`, `Grpc.Core.Api.dll`, `Google.Protobuf.dll`. diff --git a/docs/plans/hcal-capability-matrix.md b/docs/plans/hcal-capability-matrix.md new file mode 100644 index 0000000..283ec1f --- /dev/null +++ b/docs/plans/hcal-capability-matrix.md @@ -0,0 +1,166 @@ +# HCAL → modern-.NET reimplementation — capability matrix + +Feasibility map for a clean managed-.NET client that replaces the AVEVA Historian +SDK (`aahClientManaged` / HCAL). Grounded in: the real `ArchestrA.HistorianAccess` +public surface (`aahClientManaged.xml`), the recovered **2023 R2 gRPC contract**, the +existing **histsdk** reimplementation, and the event/storage analysis in +[`histevents.md`](histevents.md). + +## Legend + +**Status (histsdk today)** — ✅ implemented + live-verified · 🟗 partial · ⬜ not yet + +**Feasibility tier** +| Tier | Meaning | Effort | +|---|---|---| +| **DONE** | already working in histsdk | 0 | +| **TRIVIAL** | gRPC op known, payload already decoded or empty | XS (hrs) | +| **CAPTURE** | one instrument-and-capture of a native payload, then serialize + golden-byte test | S (days) | +| **BOUNDED** | gRPC op exists; decode one proprietary `bytes` payload | S–M | +| **HARD** | whole subsystem to reimplement | L (weeks) | +| **GATED** | blocked server-side — client effort doesn't unblock it | n/a | + +Effort = incremental work on top of histsdk's existing infrastructure (auth chain, +transport, frame/byte primitives, test harness). All non-DONE items assume the +**gRPC transport** as the foundation (clean protobuf envelope; only the inner byte +blob needs RE). + +--- + +## 1. Connection & session + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Probe / version | `TestConnection`, GetV | `*Service.GetInterfaceVersion` | ✅ | DONE | | +| Open connection (Process) | `OpenConnection` | `History.OpenConnection` (+ `ExchangeKey` auth) | ✅ | DONE | full auth chain works | +| Open connection (Event) | `OpenConnection` (Event type) | `History.OpenConnection` event mode | 🟗 | TRIVIAL | read path already opens it; flag = ConnectionType.Event | +| Close connection | `CloseConnection` | `History.CloseConnection` | ✅ | DONE | | +| Connection status | `GetConnectionStatus` | `Status.GetHistorianConsoleStatus` | ✅ | DONE | | +| Open/close **storage** connection | `OpenStorageConnection`, `CloseStorageConnection` | `Storage.OpenStorageConnection2` | ⬜ | BOUNDED | needed for any data-write path; storage-engine session | + +## 2. Reads — process data + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Raw / full history | `CreateHistoryQuery` → Start/MoveNext/End | `Retrieval.StartQuery`→`GetNextQueryResultBuffer`→`EndQuery` | ✅ | DONE | row buffer parsed | +| Aggregate (interp/avg/min/max/…) | `CreateHistoryQuery` (RetrievalMode) | same | ✅ | DONE | all 15 RetrievalModes mapped | +| At-time / value-at | (interp window) | same | ✅ | DONE | | +| Analog summary | `CreateAnalogSummaryQuery` | `Retrieval.StartQuery` (summary mode) | 🟗 | BOUNDED | mode variant of existing query | +| State summary | `CreateStateSummaryQuery` | `Retrieval.StartQuery` (state mode) | ⬜ | BOUNDED | extra row layout to decode | +| Block read | `ReadBlocks` | `Storage.LoadBlocks` | ⬜ | BOUNDED | low-level; rarely needed | + +## 3. Reads — events + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Event query | `CreateEventQuery` → Start/MoveNext/End | `Retrieval.StartEventQuery`→`GetNextEventQueryResultBuffer`→`EndEventQuery` | ✅ | DONE | rows + typed property bag parsed; CM_EVENT registration done | +| Event filters | `EventQuery.AddEventFilter` / `AddEventFilterCondition` | filter bytes in StartEventQuery request | ⬜ | BOUNDED | encode filter predicate into request buffer | + +## 4. Browse & metadata + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Tag name browse | `CreateTagQuery` → `GetTagNames` | `Retrieval.StartTagQuery`/`QueryTag` (or LikeTagnames) | ✅ | DONE | wildcard works | +| Tag metadata | `GetTagInfoByName`, `TagQuery.GetTagInfo` | `Retrieval.GetTagInfosFromName` | ✅ | DONE | | +| Extended properties (read) | `GetTagExtendedPropertiesByName` | `Retrieval.GetTagExtendedPropertiesFromName` | ⬜ | BOUNDED | TEP buffer decode | +| Localized properties (read) | `GetTagLocalizedPropertiesByName` | `Retrieval.GetTagLocalizedPropertiesFromName` | ⬜ | BOUNDED | | +| SQL passthrough | `ExecuteSqlCommand` | `Retrieval.ExecuteSqlCommand` | ⬜ | TRIVIAL | thin string-in / status-out | + +## 5. Tag configuration (writes) + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Create analog tag | `AddTag` | `History.EnsureTags` (EnsT2) | ✅ | DONE | Float/Double/Int2/Int4/UInt2/UInt4 + scaling | +| Create string/discrete tag | `AddTag` | `History.EnsureTags` | ⬜ | GATED/BOUNDED | native AddTag rejects these types server-side; needs different metadata path | +| Delete tag(s) | `DeleteTags` | `History.DeleteTags` | ✅ | DONE | | +| Rename tag(s) | `RenameTags` | (History op) | ⬜ | BOUNDED | `AllowRenameTags` param already probed | +| Add/Delete extended properties | `AddTagExtendedProperties`, `DeleteTagExtendedPropertiesByName` | `History.AddTagExtendedProperties` / `DeleteTagExtendedProperties` | ⬜ | BOUNDED | gRPC op + TEP serialize | +| Add/Delete localized properties | `AddTagLocalizedProperties`, `DeleteTagLocalizedPropertiesByName` | `History.AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` | ⬜ | BOUNDED | | + +## 6. Data writes — values + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Stream process values | `AddStreamedValue(HistorianDataValue)` | `Storage.AddStreamValues` | ⬜ | **GATED** | runtime cache only ingests from IOServer/AppServer pipelines (`129 Tag not found in cache`). Not a client bug | +| Stream **events** | `AddStreamedValue(HistorianEvent)` | `Storage.AddStreamValues` (event VTQ) | ⬜ | **CAPTURE** | full path mapped; need `CCommonArchestraEventValue::PackToVtq` blob bytes. See histevents.md | +| Non-streamed / historical insert | `AddNonStreamedValue`, `SendNonStreamedValues` | `Transaction.AddNonStreamValues(Begin/End)` | ⬜ | BOUNDED | explicit original-data insert via Transaction svc; verify ingest permission on target | +| Versioned streamed value | `AddVersionedStreamedValue` | `Storage.AddStreamValues2` | ⬜ | CAPTURE | revision flag on the VTQ | + +## 7. Revisions / edits (modify stored data) + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Insert/update/delete revision values | `AddRevisionValue(s)`, `AddRevisionValuesBegin/End` | (storage-engine / transaction path) | ⬜ | HARD | prior RE: revision-write needs the non-WCF **storage-engine pipe** (`STransactPipeClient2`), not the WCF/gRPC surface | +| Event update/delete (revise) | `HistorianEvent.Update/.Delete` | `UpdateEventStatus` (+ revised VTQ) | ⬜ | CAPTURE | RevisionVersion + Update/Delete flags in the event VTQ | + +## 8. Status & system info + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| System parameter | `GetSystemParameter` | `Status.GetSystemParameter` | ✅ | DONE | | +| Runtime parameter | `GetRuntimeParameter` | `Status.GetRuntimeParameter` | ⬜ | TRIVIAL | same shape as GetSystemParameter | +| Historian info | `GetHistorianInfo` | `Status.GetHistorianInfo` | 🟗 | BOUNDED | GETHI buffer; partially decoded (incl. EventStorageMode @ offset 514) | +| Server timezone | `GetSystemTimeZoneInfo` | `Status.GetSystemTimeZoneName` | ⬜ | TRIVIAL | | +| Historization status | `GetHistorizationStatus` | `Status` op | ⬜ | BOUNDED | | +| Store-and-forward status | `GetStoreForwardStatus` | (push events / pull GETHI) | 🟗 | HARD | currently synthesized; real read needs duplex push or a decoded pull endpoint — see store-forward plan | + +## 9. Store-and-forward (offline buffering) + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| SF buffering + replay | (implicit on write conns) | `Storage`/`Transaction` `*Snapshot` + `Forward*Snapshot` | ⬜ | HARD | full subsystem: local cache format, snapshot framing, recovery log, forward-on-reconnect. Pragmatic alt: a simpler local queue, not bit-faithful SF | +| Event SF | (event conn) | `Forward**Event**SnapshotBegin/…/End` | ⬜ | HARD | dedicated event-snapshot SF stream | +| SF parameters | Get/Set SFP | `Storage.GetSFParameter`/`SetSFParameter` | ⬜ | BOUNDED | | + +## 10. Redundancy / multi-historian + +| Capability | HCAL API | 2023 R2 gRPC op | histsdk | Tier | Notes | +|---|---|---|---|---|---| +| Tiered/redundant access, failover | `MultiHistorianAccess.*` (OpenConnectionToAll, AddSecondaries, partner watchdog, ReSyncTags) | N×single-historian sessions + client logic | ⬜ | HARD | mostly client-side orchestration over §1–§6; build last | +| Replication config | (server `aahReplication`) | — | ⬜ | GATED | server-side concern | + +--- + +## Roll-up & recommended cut line + +**Phase 0 — already DONE (✅):** probe · open/close · raw+aggregate+at-time reads · +event reads · tag browse · tag metadata · system parameter · connection status · +create/delete analog tag. This is a usable modern client **today**. + +**Phase 1 — TRIVIAL/BOUNDED, high value (S–M each):** ExecuteSqlCommand · +runtime parameter · server timezone · extended/localized property read · event +filters · summary/state-summary queries · rename tags · ext/localized property +writes · GetHistorianInfo. Each is "gRPC op exists, decode one buffer, golden-byte +test." Knocks out most of the remaining read/config surface. + +**Phase 2 — CAPTURE (one native capture each, S):** **event sending** (the headline +gap — fully mapped, one `PackToVtq` capture away) · versioned/non-streamed value +writes. Now feasible locally since the Historian is installed. + +**Defer / simplify (HARD):** store-and-forward (do a pragmatic local queue instead of +bit-faithful SF) · revision/edit writes (separate storage-engine pipe) · multi- +historian redundancy (client orchestration, build last). + +**Won't unblock from the client (GATED):** streaming **process-sample** writes +(`AddS2`) — server cache only ingests from IOServer/AppServer pipelines; confirm your +ingestion model rather than chasing this. Non-analog tag creation likely needs a +distinct server path. + +## Cross-cutting realities (apply to every non-DONE row) + +- **Inner payloads stay proprietary** even under gRPC — the `bytes` fields carry + native VTQ / CTagMetadata / event-value formats. These are **version-sensitive**; + pin to the server version probed at connect and fail closed on mismatch. +- **Validation needs a live Historian** — now available locally, which is what makes + the CAPTURE-tier items practical. +- **Support tradeoff** — you take on maintenance across Historian versions in exchange + for shedding the stock SDK's bugs (mixed-mode marshaling, WCF quirks, global state) + for the surface you cover. + +## Bottom line + +A modern-.NET HCAL replacement is **feasible and ~60–70% done** for a typical +read+browse+config+event-read workload. The remaining high-value surface is mostly +**BOUNDED/CAPTURE** (incremental, well-understood), with only store-and-forward, +revision-edit, and redundancy being genuine **HARD** subsystems — and one true wall +(**GATED** process-sample writes) that no client can remove. diff --git a/docs/plans/hcal-roadmap.md b/docs/plans/hcal-roadmap.md new file mode 100644 index 0000000..ac7ed8d --- /dev/null +++ b/docs/plans/hcal-roadmap.md @@ -0,0 +1,171 @@ +# HCAL modern-.NET client — implementation roadmap + +Ordered, actionable plan to grow **histsdk** from "reads + basic config" into a broad +HCAL replacement, built on the **2023 R2 gRPC transport**. Derived from +[`hcal-capability-matrix.md`](hcal-capability-matrix.md); event details in +[`histevents.md`](histevents.md). + +> Move to the repo's `docs/plans/` when execution starts. Each work item lands as: a +> protocol serializer/parser + golden-byte unit test + an env-gated live integration +> test against the local Historian. + +## Guiding principles + +1. **gRPC-first.** New ops go on the `RemoteGrpc` transport (clean protobuf envelope); + the inner `bytes` blob is the only thing to RE. Keep WCF as the legacy/Windows path. +2. **Two tests per op, always.** A golden-byte test (deterministic, no server) **and** a + gated live test (`HISTORIAN_GRPC_HOST` / `HISTORIAN_HOST`). No op is "done" without both. +3. **Version-pin, fail closed.** Read server version at connect; gate every byte + serializer on it; throw `ProtocolEvidenceMissingException` on mismatch — never + best-effort parse. +4. **Capture once, encode forever.** For CAPTURE-tier items, instrument one native call, + save a sanitized fixture under `fixtures/protocol/`, then implement against the fixture. +5. **Ship per milestone.** Each milestone is independently releasable. + +Effort: **S** ≈ days · **M** ≈ ~1 week · **L** ≈ weeks. Estimates are incremental on +histsdk's existing infra (auth chain, transport, frame primitives, test harness). + +--- + +## Milestone 0 — Foundation: full gRPC parity for the DONE surface (M) + +*Goal: everything already working over WCF also works over `RemoteGrpc`, so the whole +read/browse/status surface is Windows-free and the gRPC stack is the default path.* + +| ID | Work | gRPC op | Files | Verify | Effort | +|---|---|---|---|---|---| +| R0.1 | Route browse over gRPC | `Retrieval.StartTagQuery`/`QueryTag` or `GetTagInfosFromName` | `Grpc/HistorianGrpcReadOrchestrator` (+ new `…GrpcBrowseClient`), `Historian2020ProtocolDialect` | browse tags live over gRPC | S | +| R0.2 | Route tag metadata over gRPC | `Retrieval.GetTagInfosFromName` | dialect + grpc client | metadata matches WCF result | S | +| R0.3 | Route status/system-param over gRPC | `Status.GetSystemParameter`, `Status.GetHistorianConsoleStatus` | new `Grpc/HistorianGrpcStatusClient` | system param + conn status live | S | +| R0.4 | Probe over gRPC | `*.GetInterfaceVersion` | grpc clients | `ProbeAsync` Windows-free | XS | +| R0.5 | **Capture harness for gRPC payloads** | n/a | reuse `instrument-wcf-*` tooling (same byte blobs) + add a `grpc-call-dump` helper | dump any request/response `bytes` to a fixture | S | +| R0.6 | **Version gate** | server version at connect | `HistorianClientOptions`, orchestrators | mismatched version → throws | S | + +**Acceptance:** the entire Phase-0 capability set runs end-to-end over `RemoteGrpc` +(incl. Linux), no WCF on the path. 188+ unit tests green; live gRPC integration suite green. + +--- + +## Milestone 1 — Cheap surface completion (TRIVIAL/BOUNDED) (M–L total) + +*Goal: knock out the remaining read/config surface. Order = ascending payload difficulty.* + +### 1a. Trivial (XS–S each, no new payload format) +| ID | Capability | gRPC op | Notes | +|---|---|---|---| +| R1.1 | `ExecuteSqlCommandAsync` | `Retrieval.ExecuteSqlCommand` | string in → `iRetValue` + status; thin | +| R1.2 | `GetRuntimeParameterAsync` | `Status.GetRuntimeParameter` | mirror `GetSystemParameter` | +| R1.3 | `GetServerTimeZoneAsync` | `Status.GetSystemTimeZoneName` | string out | + +### 1b. Bounded (decode one `bytes` payload; S–M each) +| ID | Capability | gRPC op | Payload to decode | Depends | +|---|---|---|---|---| +| R1.4 | `GetHistorianInfoAsync` | `Status.GetHistorianInfo` | GETHI buffer (partly decoded; incl. `EventStorageMode`@514) | R0.5 | +| R1.5 | Extended-property **read** | `Retrieval.GetTagExtendedPropertiesFromName` | TEP result buffer | R0.5 | +| R1.6 | Localized-property **read** | `Retrieval.GetTagLocalizedPropertiesFromName` | localized buffer | R0.5 | +| R1.7 | Event **filters** | filter bytes in `Retrieval.StartEventQuery` | filter predicate encoding (name/op/value) | R0.5 | +| R1.8 | Analog-summary query | `Retrieval.StartQuery` (summary mode) | summary row layout | — | +| R1.9 | State-summary query | `Retrieval.StartQuery` (state mode) | state-summary row layout | — | + +### 1c. Bounded config writes (S–M each) +| ID | Capability | gRPC op | Payload | Notes | +|---|---|---|---|---| +| R1.10 | `RenameTagsAsync` | History rename op | rename request buffer | `AllowRenameTags` already probed | +| R1.11 | Extended-property **write** | `History.AddTagExtendedProperties` (+ groups) / `DeleteTagExtendedProperties` | TEP serialize | mirror analog CTagMetadata discipline | +| R1.12 | Localized-property **write** | `History.AddTagLocalizedProperties` / `DeleteTagLocalizedProperties` | localized serialize | | +| R1.13 | Non-analog tag create (string/discrete) | `History.EnsureTags` | distinct CTagMetadata variant | ⚠ native AddTag rejected some types — confirm server path first; may be GATED | + +**Acceptance:** read + browse + metadata + system/status + property R/W + summaries + +event-filtered reads + rename all live-verified over gRPC. + +--- + +## Milestone 2 — Event sending (CAPTURE) (S–M) ← headline gap + +*Goal: `SendEventAsync(HistorianEvent)`. Path fully mapped in histevents.md; one capture away.* + +| ID | Work | Detail | +|---|---|---| +| R2.1 | Capture the event value blob | Instrument `CCommonArchestraEventValue::PackToVtq` (or dump the VTQ value bytes) on a live `AddStreamedValue(HistorianEvent)`; save sanitized fixture | +| R2.2 | `HistorianEventWriteProtocol` | Serialize header (`ReceivedTime, EventType, EventTime, Id, RevisionVersion, IsUpdate/IsDelete, Namespace`) + typed property bag — **inverse of `HistorianEventRowProtocol`** (reuse typemarkers `0x02/0x10/0x18/0x31/0x43/…`) | +| R2.3 | Event write orchestrator | Open **Event** connection (write mode) → register CM_EVENT (already have) → `Storage.AddStreamValues` with the event VTQ | +| R2.4 | Public API | `HistorianClient.SendEventAsync(HistorianEvent)` (+ `HistorianEvent` model: Type, EventTime, property bag) | +| R2.5 | Round-trip test | Send an event → read it back via `StartEventQuery` / `v_AlarmEventHistory2`; golden-byte on R2.2 | + +**Acceptance:** an event sent from histsdk appears in the historian and is read back with +matching Type + properties. **Now practical** — Historian is installed locally. + +--- + +## Milestone 3 — Historical / non-streamed value writes (BOUNDED) (M) + +*Goal: insert original historical VTQs (backfill), the path that is NOT the gated cache push.* + +| ID | Work | gRPC op | +|---|---|---| +| R3.1 | Decode non-streamed VTQ packet | `Transaction.AddNonStreamValuesBegin/AddNonStreamValues/End` | +| R3.2 | `AddHistoricalValuesAsync` | batched begin→values→end | +| R3.3 | Ingest-permission validation | confirm the target accepts original-data insert (distinct from `AddS2` cache wall) | + +**Acceptance:** historical points inserted and read back. Document clearly where this +differs from (gated) streaming sample writes. + +--- + +## Milestone 4 — HARD subsystems (deferred / optional) (L each) + +Only if the use case demands them. Each is a real subsystem, not an op. + +| ID | Capability | Approach | Risk | +|---|---|---|---| +| R4.1 | Store-and-forward | **Pragmatic local queue** (durable outbox + replay on reconnect) rather than bit-faithful SF cache + `Forward*Snapshot`. Faithful SF = decode SF cache format + snapshot framing + recovery log | high; consider "good enough" | +| R4.2 | Revision / edit writes | `AddRevisionValue(s)` go via the **non-WCF storage-engine pipe** (`STransactPipeClient2`) — separate transport RE | high | +| R4.3 | Real store-forward **status** | duplex push (`SetStoreForwardEvent`) or a decoded pull endpoint — see store-forward plan | medium | +| R4.4 | Multi-historian / redundancy | client-side orchestration over N single-historian sessions (failover, ReSyncTags, partner watchdog) — build last | medium | + +--- + +## Won't-do from the client (GATED) + +- **Streaming process-sample writes** (`AddStreamedValue(HistorianDataValue)` / `AddS2`): + runtime cache only ingests from configured IOServer/AppServer pipelines. Confirm your + ingestion architecture instead of pursuing this. + +--- + +## Cross-cutting workstreams (run alongside all milestones) + +- **CW-1 Capture tooling** (enables R0.5, R1.x, R2.1): one reusable "call op → dump + request/response `bytes` → sanitized fixture" path. Highest leverage — do first. +- **CW-2 Version compatibility:** matrix of tested Historian versions; serializers keyed + by version; CI gate. +- **CW-3 Cross-platform CI:** run the gRPC suite on Linux/macOS (transport is portable; + explicit-cred auth path). +- **CW-4 Fixtures discipline:** every new op ships a `fixtures/protocol//` golden file; + sanitize hostnames/tags/GUIDs before commit. +- **CW-5 Public API shape:** keep the modern surface (async, `IAsyncEnumerable`, + cancellation, options record, DI-friendly) consistent as the surface grows. + +--- + +## Sequencing (critical path) + +``` +CW-1 capture tooling ─┐ +M0 gRPC parity ───────┼─→ M1 cheap surface ─→ M2 event send ─→ M3 historical writes ─→ (M4 optional) +R0.6 version gate ────┘ +``` + +Recommended first sprint: **CW-1 + M0 (R0.1–R0.6)** → a fully Windows-free, version-safe +gRPC client at today's capability. Second sprint: **M1a + M2** (cheap wins + the headline +event-send). M3/M4 as demand dictates. + +## One-glance status + +| Milestone | Tier | Effort | Value | When | +|---|---|---|---|---| +| M0 gRPC parity + capture tooling | foundation | M | unblocks everything, Windows-free | **now** | +| M1 cheap surface | TRIVIAL/BOUNDED | M–L | most remaining read/config | next | +| M2 event send | CAPTURE | S–M | headline write capability | next | +| M3 historical writes | BOUNDED | M | backfill | on demand | +| M4 SF / revisions / redundancy | HARD | L×N | parity completeness | defer | diff --git a/docs/plans/histevents.md b/docs/plans/histevents.md new file mode 100644 index 0000000..c177111 --- /dev/null +++ b/docs/plans/histevents.md @@ -0,0 +1,299 @@ +# How a HistorianEvent reaches the Historian DB files + +Living analysis doc. Traces an event end-to-end: client API → wire → server +storage backend (SQL **Database** vs history **Blocks** `.dat`) → read-back. + +Evidence base: 2023 R2 `aahClientManaged.dll` (decompiled `ArchestrA.HistorianAccess`, +`HistorianEvent`, `HistorianEventPropertyType`), native `aahStorage.exe` (string +analysis), the recovered gRPC + CloudHistorian contracts, and the histsdk read-side +reverse-engineering (CM_EVENT registration + event-row parser). + +Status legend: ✅ proven (from binary) · 🔶 strong inference · ❓ open. + +--- + +## TL;DR + +An event is **not a distinct wire message**. The client turns each `HistorianEvent` +into a `HistorianDataValue` of type `Event` against the built-in **`CM_EVENT`** tag, +marshals it into a native VTQ, and **streams it like any tag value** on a dedicated +*Event* connection. Events are batched into an opaque serialized **event data packet** +and delivered (with their own store-and-forward "event snapshot" path). On the server +they are persisted into **one of two backends, chosen by a server-configured +`EventStorageMode`**: a SQL **Database**, or the history **Blocks** (`.dat`) files. + +``` +HistorianEvent (Type + typed property bag) + └─(AddStreamedValue)→ HistorianDataValue{Type=Event, TagKey=CM_EVENT, EventTime, Q=192} + └─ HistorianEvent.PackToVtq → CCommonArchestraEventValue::PackToVtq (native value blob) + └─ HISTORIAN_VALUE2 (44B; blob ptr @+33) → HistorianClient.AddHistorianValue → queue + └─ flush → EnqueueEventDataPacket{ byte[] SerializedBytes } (batched VTQs) + └─ SERVER: aahEventStorage.exe (InSQLEventSystem) + per-client event-tag pipeline → recovery-log WAL → backend: + • Blocks → elastic snapshot → frozen → history .dat (Circular/Permanent) + • Database → ArchestrAEvents.EventStorage.Contract assembly → SQL (A2ALMDB) + → EventReplication (redundant historians) + └─ (offline) → store-and-forward → ForwardEventSnapshotBegin/…/End on reconnect + read-back: Retr.StartEventQuery / SQL provider views (Events, v_AlarmEventHistory2, v_EventSnapshot) +``` + +--- + +## 1. The `HistorianEvent` object ✅ + +Decompiled `ArchestrA.HistorianEvent` — a structured header plus a **typed property bag**: + +- **Header fields:** `ID`/`Id` (Guid), `Type`/`EventType` (string, e.g. `"Alarm.Set"`, + `"User.Write"`), `EventTime` (DateTime), `ReceivedTime`, `Severity` (ushort), + `Priority` (ushort), `IsAlarm`, `IsSilenced`, `System`, `Source`, `Source_Name`, + `Area`, `Namespace`, `DisplayText`. +- **Revision fields:** `RevisionVersion` (ushort), `Delete` (bool), `Update` (bool) — + events are revisable (see §4 UpdateEventStatus). +- **Property bag:** `AddProperty(name, value, HistorianEventPropertyType, …)` with typed + overloads. `HistorianEventPropertyType` (alphabetical enum): + `Blob, Boolean, Byte, Date, DateTime, Decimal, Double, Duration, Float, Guid, Hex, + Int, Integer, Long, Short, String, Time, UnsignedByte, UnsignedInt, UnsignedLong, + UnsignedShort, Undefined`. + +These map onto the wire property-bag the histsdk **read** parser already decodes +(`HistorianEventRowProtocol`): typemarkers `0x02` Boolean, `0x10` Guid, `0x18` FILETIME, +`0x31` Int32, `0x43` UTF-16 string, … — i.e. the write enum and the read typemarkers are +two views of the same typed-value format. The event-send serialization is the inverse of +that read parser. + +--- + +## 2. Client send path — an event becomes a streamed VTQ ✅ + +From decompiled `ArchestrA.HistorianAccess` (line refs into the decompile): + +1. **Open an Event connection.** `HistorianConnectionArgs.ConnectionType = + HistorianConnectionType.Event`, `ReadOnly = false` (sample `Step10.SendEvents`). +2. **Default event tag.** `CreateDefaultEventTag()` (`:3006`) registers tag `CM_EVENT` / + "AnE Event" / `TagDataType = Event` and stores `eventTagHandle`. Same CM_EVENT + registration histsdk reverse-engineered (RTag2 + EnsT2; tag id + `353b8145-5df0-4d46-a253-871aef49b321`). +3. **Wrap as VTQ.** `AddStreamedValue(HistorianEvent)` (`:3123`): + ```csharp + historianDataValue.objValue = historianEvent; // header + property bag + historianDataValue.DataValueType = HistorianDataType.Event; + historianDataValue.TagKey = eventTagHandle; // CM_EVENT + historianDataValue.StartDateTime = historianEvent.EventTime; + historianDataValue.OpcQuality = 192; + return AddStreamedValue((ConnectionIndex)1, historianDataValue, false, out error); // 1=Event + ``` +4. **Marshal + queue.** The private `AddStreamedValue` (`:3173`): + - builds a 44-byte native `HISTORIAN_VALUE2` (`InitBlockUnaligned(…,0,44)`), + - `HistorianAccessUtil.ConvertManagedStructToUnmanagedStruct(value, &HV2, bVersioned…)` + — its `case HistorianDataType.Event` (`HistorianAccessUtil:89`) calls + **`HistorianEvent.PackToVtq(out byte[])`** to produce the event value blob, whose pointer + is placed at `HISTORIAN_VALUE2+33` (freed after send; offset 33 is the value-union pointer + used for Event/String types), + - `HistorianClient.AddHistorianValue(client, &HV2, &err)` (`:3209`) queues the VTQ into + the native delivery buffer and returns immediately. + +So an event uses the **same streaming machinery as a process value**; only `DataValueType` +(`Event`) and the target tag (`CM_EVENT`) differ. + +### 2a. Event value serialization — `HistorianEvent.PackToVtq` ✅/🔶 + +`HistorianEvent.PackToVtq` (`HistorianEvent:1392`) populates a native +**`CCommonArchestraEventStruct`** then hands it to the **native** packer +`CCommonArchestraEventValue::PackToVtq(…, 192, 192, vtq)` (Q=192), associated with the +built-in `EVENT_TAGID` / `EVENT_TAGNAME` (`CTagMetadata.CommonArchestraEvent`). The actual +byte layout is produced in C++ — **not visible in managed code** — so pinning exact write +bytes needs a wire/IL capture, exactly as the read side did. But the **field set + order** +the managed code writes into the struct is now known: + +``` +SetReceivedTime (uint64 FILETIME, from UniqueTime.GetUniqueFileTime — unique/monotonic) +SetEventType (wchar* string, e.g. "Alarm.Set") +SetEventTime (uint64 FILETIME, from EventTime) +SetId (GUID) +SetRevisionVersion (uint16) +SetIsUpdate (bool) ← revision flags +SetIsDelete (bool) +Namespace (string, trimmed, non-printable-validated) +…then the typed property bag: Dictionary> +``` + +This matches `HistorianEventRowProtocol` on read: the property bag is name→(type,value) with +the same typed-value encoding (typemarkers `0x02/0x10/0x18/0x31/0x43/…`). So a managed +event-send serializer is tractable: emit the header struct fields above, then the typed +property bag in the read parser's format. The remaining unknown is only the exact native +framing offsets — best obtained by capturing one `PackToVtq` output, then golden-byte testing. + +--- + +## 3. Event transport / delivery pipeline ✅ (CloudHistorian + gRPC contracts) + +Events have a **dedicated, batched** connection + delivery pipeline, distinct from tag data +but structurally parallel: + +| Stage | Event op | Tag-data analogue | +|---|---|---| +| Open connection | `OpenEventConnection2 { byte[] ClientInfo } → { byte[] ServerInfo }` | OpenConnection | +| Send batch | `EnqueueEventDataPacket { byte[] SerializedBytes }` | `EnqueueTagDataPacket { byte[] SerializedBytes }` | +| Store-and-forward | `ForwardEventSnapshotBegin / ForwardEventSnapshot / ForwardEventSnapshotEnd` | `ForwardSnapshot…` | +| Revise | `UpdateEventStatus` | (revision write) | + +Key point: the **event data packet is an opaque serialized byte buffer** (`SerializedBytes`, +DataMember `d`) — the queued event VTQs batched together, exactly the same envelope shape as +the tag data packet. On-prem this is what the storage-streaming op (`AddStreamValues`) +carries; in the cloud variant it is `EnqueueEventDataPacket`. + +Validation surfaced via error codes: `InvalidAlarmEventPropertyLength=212`, +`AlarmEventPropertyHasNonPrintableChar=214`, `AlarmEventPropertyHasInvalidSpecialChar=215`, +`AlarmEventPropertyNameIsAReservedName=216` — the server validates alarm/event property +names + values on ingest. + +Offline → events spool to the **store-and-forward** cache and replay as **event snapshots** +(`ForwardEventSnapshot*`) on reconnect — a separate SF stream from tag-data snapshots. + +--- + +## 4. Revisions / updates ✅ + +`HistorianEvent.Update` / `.Delete` / `.RevisionVersion` + the contract's +`UpdateEventStatus` op mean events are not write-once: an event can be re-sent to update or +delete a previously stored event (e.g. alarm acknowledge/clear), bumping `RevisionVersion`. + +--- + +## 5. The storage-backend switch — `EventStorageMode` ✅ + +The client reads the server's event-storage backend from `HISTORIAN_INFO` **byte offset 514** +(`HistorianAccess` `:5715`): + +```csharp +EventStorageMode = (info[514] == -1) ? Unsupported + : (info[514] == 0) ? Database // SQL Server + : Blocks; // history .dat blocks +``` + +`HistorianEventStorageMode ∈ { Database, Blocks, Unsupported }`. The destination is a +**server** decision; the client streams the same VTQ regardless. + +--- + +## 6. Server side — where it lands ✅ (confirmed on the live local install) + +The server-side event component is **`aahEventStorage.exe`** (service `InSQLEventSystem`, +"AVEVA Historian Event System"; plus `aahEventSvc.exe`), at +`…\Wonderware\Historian\x64\aahEventStorage.exe`. Its string table maps the full pipeline: + +``` +event packets / forwarded snapshots → per-client "event tag pipeline" → batch enqueue + → Event Storage Recovery Log (WAL; "enqueuing N events to log", + path SystemParameter EventStorageLogPath = C:\Historian\Data\Logs\EventStorage) + → persist to the active backend: + Block Storage ("Enabled Block Storage for events") → history .dat blocks + Database ("storing N events in database") → SQL via loaded managed + assembly ArchestrAEvents.EventStorage.Contract.EventStorageDatabaseConnection + (e.g. ";Initial Catalog=A2ALMDB;Integrated Security=true;Encrypt=True;…") + → also fed to EventReplication (aahReplication.exe) for redundant historians +``` + +So persistence is **pluggable** (a loadable connection assembly) and dual-mode, guarded by a +recovery log. Which backend is live depends on configuration (the `EventStorageMode` of §5). + +### This historian = **Block storage** (verified) +- `C:\Historian\Data\Circular` holds **527 `.dat` history blocks** (`Permanent` empty); the + EventStorage recovery log dir exists. `aahEventStorage` logs `"Enabled Block Storage for + events"`. +- SDK-shape alarm/events are present and retrievable: `Runtime.dbo.v_AlarmEventHistory2` + returns 224 rows over the last 30 days. +- `A2ALMDB` (the System-Platform alarm DB the connection string references) is **not present** + here — that path is only used when integrated with AVEVA System Platform alarming. Absent + it, ArchestrA events land in **blocks**, exactly as `aahStorage.exe` advertises (`"Stores + ArchestrA Event Data"`, snapshot→block). + +### The SQL surface is **provider-backed views, not physical tables** ✅ +In `Runtime`, the rich event objects are **views with NULL `OBJECT_DEFINITION`** — i.e. the +historian's OLE DB History provider exposes them as virtual/extension tables that read the +block store, *not* stored T-SQL: +- `Events`, `v_EventHistory`, `v_EventSnapshot`, `v_EventStringSnapshot`, **`v_AlarmEventHistory2`** + (columns: `EventStampUTC`, `AlarmState`, `TagName`, `Description`, `Area`, `Type`, `Value`, + `Priority`, `Category`, `Provider`, `Operator`, `DomainName`, `UserFullName`, `MilliSec`, …) + — these are the read-back of the SDK alarm/event property bag. + +So `SELECT … FROM Events` (and `v_AlarmEventHistory2`) is **the provider reading the block +store**, which is why the handoff could query events even though they live in `.dat` blocks. + +### Database-mode physical store +When events ARE stored in SQL (Database mode / A2ALMDB integration), the writer is the loaded +`ArchestrAEvents.EventStorage.Contract` connection assembly doing batched inserts ("storing N +events in database", "creating event storage database connection role"). The exact table +schema there is the A2ALMDB alarm schema (not present on this box to dump). + +### Read-back ✅ +Uniform regardless of backend: `Retr.StartEventQuery` → `GetNextEventQueryResultBuffer` +(provider) surfaces events from wherever they were stored — so histsdk `ReadEventsAsync` is +mode-agnostic. Engine filter note: `"EventTime filtering can only be specified through +StartDateTime and EndDateTime"`. + +## 6b. Two different "event" subsystems — don't conflate ✅ + +| | Classic event **detectors** | ArchestrA **alarms/events** (the SDK path) | +|---|---|---| +| What | server-side detectors watching tag conditions | client-streamed `HistorianEvent` (alarms, user events) | +| Config/store | `Runtime.dbo._EventTag` (TagName, DetectorTypeKey, DetectorString, Action*, ScanRate, Edge, Priority) | CM_EVENT / CommonArchestraEvent tag | +| History | **physical** `BASE TABLE Runtime.dbo.EventHistory` (`EventLogKey, TagName, DateTime, DetectDateTime, Edge`); 30 rows | block store (or A2ALMDB), surfaced via `v_AlarmEventHistory2` / `v_EventSnapshot` | +| Source | evaluated by the server | sent via `AddStreamedValue(HistorianEvent)` | + +`AddStreamedValue(HistorianEvent)` feeds the **right column** (ArchestrA alarms/events) — it is +**not** the classic `EventHistory` detector log. + +--- + +## 7. Relationship to histsdk + +- histsdk implements event **reads** only (`ReadEventsAsync` via `StartEventQuery`); its + CM_EVENT EnsT2/RTag2 dance is read-subscription registration. +- Event **writing** is unimplemented but viable. Chain to replicate: Event-type connection → + register CM_EVENT (done) → serialize `HistorianEvent` (header + typed property bag) into the + event-VTQ value blob (inverse of `HistorianEventRowProtocol`) → batch into an event data + packet → stream via `AddStreamValues` (2023 R2 gRPC: `StorageService.AddStreamValues`). + +--- + +## Open threads + +- 🔶 Event value blob: field set/order known (§2a); **exact native framing** still needs one + `CCommonArchestraEventValue::PackToVtq` output capture + golden-byte test (mirror the read-side + `HistorianEventRowProtocol` reverse-engineering). Now feasible locally — the live historian is + installed, so the same instrument-and-capture approach used for reads applies. +- ❓ `EnqueueEventDataPacket.SerializedBytes` packet framing (header + N event VTQs batched). +- ✅ Database-mode store: server writer is `aahEventStorage.exe` loading the managed + `ArchestrAEvents.EventStorage.Contract` connection assembly; SQL retrieval surface is the + provider-backed `Events` / `v_AlarmEventHistory2` / `v_EventSnapshot` views (NULL T-SQL def). + This box runs **Block storage** (A2ALMDB absent). A2ALMDB physical schema still un-dumped (needs + a System-Platform-integrated box). +- ✅ `ArchestraEvent` vs `CommonArchestraEvent`: send path packs **CommonArchestraEvent** via + `EVENT_TAGID`; both are event-tag schemas in `CTagMetadata` (the server stores either). +- ❓ `UpdateEventStatus` wire payload for `Update`/`Delete` revisions. +- ❓ `EventStorage` recovery-log (`C:\Historian\Data\Logs\EventStorage`) on-disk format (WAL). +- 🔶 Decompile `ArchestrAEvents.EventStorage.Contract.dll` (managed) for the exact DB insert + contract/schema — locate it (not under `…\Wonderware\Historian`; check GAC / Framework\Bin). + +--- + +## Changelog +- Rev 4 (live local install): confirmed server side. `aahEventStorage.exe` (`InSQLEventSystem`) + is the event store engine — per-client event-tag pipeline → recovery-log WAL → Block storage + OR SQL (loadable `ArchestrAEvents.EventStorage.Contract` assembly) → EventReplication. This box + uses **Block storage** (527 `.dat` in `C:\Historian\Data\Circular`; A2ALMDB absent). SQL + `Events`/`v_AlarmEventHistory2`/`v_EventSnapshot` are **provider-backed views over the blocks** + (NULL `OBJECT_DEFINITION`), not physical tables — `v_AlarmEventHistory2` (224 rows/30d) is the + SDK-event read surface. Distinguished the classic event-detector subsystem (`_EventTag` → + physical `EventHistory`) from the ArchestrA alarm/event path (the SDK's target). +- Rev 3: event value serialization pinned to native `CCommonArchestraEventValue::PackToVtq` + via managed `HistorianEvent.PackToVtq`; documented the `CCommonArchestraEventStruct` field + set/order (ReceivedTime, EventType, EventTime, Id, RevisionVersion, IsUpdate/IsDelete, + Namespace, typed property bag) and the path to a managed send serializer. +- Rev 2: HistorianEvent structure + HistorianEventPropertyType enum; client marshaling + (HISTORIAN_VALUE2 / ConvertManagedStructToUnmanagedStruct / AddHistorianValue); dedicated + event pipeline (OpenEventConnection2 / EnqueueEventDataPacket / ForwardEventSnapshot / + store-and-forward); revisions (Update/Delete/UpdateEventStatus); Blocks-mode clarified + (events = generic VTQ snapshots, no event-specific block code). +- Rev 1: client send path, EventStorageMode switch, Blocks/Database backends, read-back. From 6b892b69ba4bad13304f16c3bebaddaba95840a7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 14:48:52 -0400 Subject: [PATCH 3/6] R0.6: fail-closed server interface-version gate at connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns the previously-discarded GetInterfaceVersion result into a connect-time version pin. The native buffers carried in the WCF/MDAS body (and in the 2023 R2 gRPC bytes fields) are framed per native interface version; parsing them against an unexpected version risks silent misinterpretation, so we throw rather than best-effort parse. - HistorianServerVersionGate + HistorianServiceInterface: evidence-based supported versions discovered from a live Historian 2020 server (product 20.0.000) via the wcf-probe command — History=11, Retrieval=4, Transaction=2. Status' GetInterfaceVersion returns 0, so Status is reachability-only. - HistorianClientOptions.VerifyServerInterfaceVersion (default true) — bypass knob for bringing up a server whose reported integers aren't yet captured (e.g. a 2023 R2 gRPC endpoint carrying the same proven 2020 buffers). - Wired into both transports' connect paths: WCF history (auth-chain helper) + retrieval (read orchestrator), and gRPC history + retrieval. - Mismatch throws ProtocolEvidenceMissingException naming reported/expected version and the bypass knob. 10 new unit tests (198 total green). Verified the gate does not regress the proven WCF read path: a live read against the local 2020 server reaches past the gate (Retr=4 matches) — the only live failures are a pre-existing environmental read timeout (OperationCanceledException), identical with and without this change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Grpc/HistorianGrpcReadOrchestrator.cs | 12 ++- .../HistorianClientOptions.cs | 12 +++ .../HistorianServerVersionGate.cs | 93 ++++++++++++++++++ .../Wcf/HistorianWcfAuthChainHelper.cs | 3 +- .../Wcf/HistorianWcfReadOrchestrator.cs | 6 +- .../HistorianServerVersionGateTests.cs | 98 +++++++++++++++++++ 6 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 src/AVEVA.Historian.Client/HistorianServerVersionGate.cs create mode 100644 tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs index d37241c..72d2d5d 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcReadOrchestrator.cs @@ -163,7 +163,9 @@ internal sealed class HistorianGrpcReadOrchestrator Guid contextKey = Guid.NewGuid(); var historyClient = new GrpcHistory.HistoryService.HistoryServiceClient(connection.Channel); - historyClient.GetInterfaceVersion(new GrpcHistory.GetInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); + GrpcHistory.GetInterfaceVersionResponse historyVersion = historyClient.GetInterfaceVersion( + new GrpcHistory.GetInterfaceVersionRequest(), connection.Metadata, Deadline(), cancellationToken); + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, historyVersion.UiVersion, _options); HistorianNativeHandshake.RunTokenRounds( (handle, wrapped, _) => @@ -210,7 +212,9 @@ internal sealed class HistorianGrpcReadOrchestrator CancellationToken cancellationToken) { var retrievalClient = new GrpcRetrieval.RetrievalService.RetrievalServiceClient(connection.Channel); - retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + GrpcRetrieval.GetRetrievalInterfaceVersionResponse retrievalVersion = retrievalClient.GetRetrievalInterfaceVersion( + new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, retrievalVersion.UiVersion, _options); byte[] requestBuffer = HistorianDataQueryProtocol.SerializeFullHistoryRequest(request); uint queryHandle = StartQuery(retrievalClient, clientHandle, requestBuffer, "raw", cancellationToken); @@ -260,7 +264,9 @@ internal sealed class HistorianGrpcReadOrchestrator CancellationToken cancellationToken) { var retrievalClient = new GrpcRetrieval.RetrievalService.RetrievalServiceClient(connection.Channel); - retrievalClient.GetRetrievalInterfaceVersion(new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + GrpcRetrieval.GetRetrievalInterfaceVersionResponse retrievalVersion = retrievalClient.GetRetrievalInterfaceVersion( + new GrpcRetrieval.GetRetrievalInterfaceVersionRequest(), null, Deadline(), cancellationToken); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, retrievalVersion.UiVersion, _options); HistorianDataQueryRequest request = HistorianWcfReadOrchestrator.BuildAggregateQueryRequest(tag, startUtc, endUtc, mode, interval); byte[] requestBuffer = HistorianDataQueryProtocol.SerializeFullHistoryRequest(request); diff --git a/src/AVEVA.Historian.Client/HistorianClientOptions.cs b/src/AVEVA.Historian.Client/HistorianClientOptions.cs index afd7174..23fc53e 100644 --- a/src/AVEVA.Historian.Client/HistorianClientOptions.cs +++ b/src/AVEVA.Historian.Client/HistorianClientOptions.cs @@ -62,4 +62,16 @@ public sealed class HistorianClientOptions /// true the server certificate chain is not validated. Default false. /// public bool GrpcUseTls { get; init; } + + /// + /// When true (default) the SDK verifies, at connect time, that the Historian server + /// reports the native interface versions its byte serializers were built against + /// (History=11, Retrieval=4, Transaction=2 — evidence from a live AVEVA Historian 2020 + /// server). A mismatch throws rather than + /// risk misparsing version-framed native buffers. Set false only when you have + /// independently confirmed wire compatibility with a different server version — e.g. + /// when bringing up a 2023 R2 gRPC server whose reported interface integers have not yet + /// been captured. See . + /// + public bool VerifyServerInterfaceVersion { get; init; } = true; } diff --git a/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs b/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs new file mode 100644 index 0000000..aa05f3c --- /dev/null +++ b/src/AVEVA.Historian.Client/HistorianServerVersionGate.cs @@ -0,0 +1,93 @@ +namespace AVEVA.Historian.Client; + +/// +/// Identifies a versioned native Historian service interface whose reported interface +/// version is validated at connect time by . +/// +internal enum HistorianServiceInterface +{ + History, + Retrieval, + Status, + Transaction +} + +/// +/// Fail-closed check (roadmap item R0.6) that a Historian server reports the native +/// interface version this SDK's byte serializers were built against. +/// +/// The opaque native buffers carried inside the WCF/MDAS message body — and, on 2023 R2, +/// inside the gRPC bytes fields — are framed per native interface version. Parsing +/// them against an unexpected version risks silent misinterpretation, so per the +/// "version-pin, fail closed" principle this throws +/// rather than best-effort parsing. +/// +/// Supported versions are evidence-based, discovered from a live AVEVA Historian 2020 +/// server (product 20.0.000) via the reverse-engineering wcf-probe command: +/// +/// History (Hist) interface version = 11 +/// Retrieval (Retr) interface version = 4 +/// Transaction (Trx) interface version = 2 +/// +/// The Status (Stat) service's GetInterfaceVersion returns 0 (not a real +/// version), so the Status interface is validated for reachability only, never value. +/// +/// A 2023 R2 gRPC server may report different integers even though it carries the same +/// proven 2020 native buffers; until those integers are captured, point such a server at +/// this gate with set to +/// . +/// +internal static class HistorianServerVersionGate +{ + public const uint HistoryInterfaceVersion = 11; + public const uint RetrievalInterfaceVersion = 4; + public const uint TransactionInterfaceVersion = 2; + + /// + /// True when the service interface reports a meaningful version that should be matched. + /// Status is reachability-only (its GetInterfaceVersion returns 0). + /// + public static bool IsValueGated(HistorianServiceInterface service) => service switch + { + HistorianServiceInterface.History => true, + HistorianServiceInterface.Retrieval => true, + HistorianServiceInterface.Transaction => true, + HistorianServiceInterface.Status => false, + _ => false + }; + + /// The interface version this SDK's serializers target for a value-gated service. + public static uint ExpectedVersion(HistorianServiceInterface service) => service switch + { + HistorianServiceInterface.History => HistoryInterfaceVersion, + HistorianServiceInterface.Retrieval => RetrievalInterfaceVersion, + HistorianServiceInterface.Transaction => TransactionInterfaceVersion, + _ => throw new ArgumentOutOfRangeException(nameof(service), service, "Service interface is not value-gated.") + }; + + /// + /// Throws when version verification is enabled + /// and the server's reported interface version differs from the version this SDK targets. + /// No-op when is + /// , when the service is not value-gated (Status), or on a match. + /// + public static void Validate(HistorianServiceInterface service, uint reportedVersion, HistorianClientOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.VerifyServerInterfaceVersion || !IsValueGated(service)) + { + return; + } + + uint expected = ExpectedVersion(service); + if (reportedVersion == expected) + { + return; + } + + throw new ProtocolEvidenceMissingException( + $"{service} interface version {reportedVersion} (this SDK's serializers target version {expected}); " + + $"set {nameof(HistorianClientOptions)}.{nameof(HistorianClientOptions.VerifyServerInterfaceVersion)}=false to bypass at your own risk"); + } +} diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs index ed0a52e..5031b24 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfAuthChainHelper.cs @@ -45,7 +45,8 @@ internal static class HistorianWcfAuthChainHelper ICommunicationObject historyChannelCo = (ICommunicationObject)historyChannel; try { - historyChannel.GetInterfaceVersion(out _); + historyChannel.GetInterfaceVersion(out uint historyVersion); + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, historyVersion, options); RunValClRounds(historyChannel, contextKey, options, cancellationToken); byte[] open2Request = HistorianNativeHandshake.BuildOpenConnection3Request(options.Host, contextKey, connectionMode); diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs b/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs index 7cb44be..2638cb6 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianWcfReadOrchestrator.cs @@ -187,7 +187,8 @@ internal sealed class HistorianWcfReadOrchestrator ICommunicationObject retrievalChannelCo = (ICommunicationObject)retrievalChannel; try { - retrievalChannel.GetInterfaceVersion(out _); + retrievalChannel.GetInterfaceVersion(out uint retrievalVersion); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, retrievalVersion, _options); uint isAllowedReturn = retrievalChannel.IsOriginalAllowed(clientHandle, out bool isAllowed); if (isAllowedReturn != 0 || !isAllowed) @@ -289,7 +290,8 @@ internal sealed class HistorianWcfReadOrchestrator ICommunicationObject retrievalChannelCo = (ICommunicationObject)retrievalChannel; try { - retrievalChannel.GetInterfaceVersion(out _); + retrievalChannel.GetInterfaceVersion(out uint retrievalVersion); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, retrievalVersion, _options); uint isAllowedReturn = retrievalChannel.IsOriginalAllowed(clientHandle, out bool isAllowed); if (isAllowedReturn != 0 || !isAllowed) diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs new file mode 100644 index 0000000..9c3a98e --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/HistorianServerVersionGateTests.cs @@ -0,0 +1,98 @@ +namespace AVEVA.Historian.Client.Tests; + +/// +/// Unit coverage for the R0.6 connect-time interface-version gate. The supported versions +/// (History=11, Retrieval=4, Transaction=2) are evidence-based, captured from a live AVEVA +/// Historian 2020 server via the reverse-engineering wcf-probe command. +/// +public sealed class HistorianServerVersionGateTests +{ + private static HistorianClientOptions Options(bool verify = true) => new() + { + Host = "histserver", + IntegratedSecurity = true, + VerifyServerInterfaceVersion = verify + }; + + [Fact] + public void VerifyServerInterfaceVersion_DefaultsToTrue() + { + Assert.True(new HistorianClientOptions { Host = "h" }.VerifyServerInterfaceVersion); + } + + [Fact] + public void Validate_MatchingVersion_DoesNotThrow() + { + // Each value-gated service accepts exactly the version this SDK targets. + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, HistorianServerVersionGate.HistoryInterfaceVersion, Options()); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, HistorianServerVersionGate.RetrievalInterfaceVersion, Options()); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Transaction, HistorianServerVersionGate.TransactionInterfaceVersion, Options()); + } + + [Fact] + public void Validate_MismatchedVersion_ThrowsProtocolEvidenceMissing() + { + // (service, wrong version) cases — one below and one above each expected value. + (HistorianServiceInterface Service, uint Version)[] cases = + [ + (HistorianServiceInterface.History, 10u), + (HistorianServiceInterface.History, 12u), + (HistorianServiceInterface.Retrieval, 3u), + (HistorianServiceInterface.Retrieval, 5u), + (HistorianServiceInterface.Transaction, 1u), + ]; + + foreach ((HistorianServiceInterface service, uint version) in cases) + { + ProtocolEvidenceMissingException ex = Assert.Throws( + () => HistorianServerVersionGate.Validate(service, version, Options())); + + // The message must name the reported version, the expected version, and the bypass knob. + Assert.Contains(version.ToString(System.Globalization.CultureInfo.InvariantCulture), ex.Operation); + Assert.Contains(HistorianServerVersionGate.ExpectedVersion(service).ToString(System.Globalization.CultureInfo.InvariantCulture), ex.Operation); + Assert.Contains(nameof(HistorianClientOptions.VerifyServerInterfaceVersion), ex.Operation); + } + } + + [Fact] + public void Validate_VerificationDisabled_NeverThrows() + { + // A wildly wrong version is tolerated when the operator opts out (e.g. bringing up a + // 2023 R2 gRPC server whose reported integers have not yet been captured). + HistorianServerVersionGate.Validate(HistorianServiceInterface.History, 999u, Options(verify: false)); + HistorianServerVersionGate.Validate(HistorianServiceInterface.Retrieval, 0u, Options(verify: false)); + } + + [Theory] + [InlineData(0u)] + [InlineData(7u)] + [InlineData(999u)] + public void Validate_StatusService_IsReachabilityOnly_NeverThrows(uint anyVersion) + { + // Status' GetInterfaceVersion returns 0 on the live server; it is not value-gated. + HistorianServerVersionGate.Validate(HistorianServiceInterface.Status, anyVersion, Options()); + Assert.False(HistorianServerVersionGate.IsValueGated(HistorianServiceInterface.Status)); + } + + [Fact] + public void IsValueGated_HistoryRetrievalTransaction_AreGated() + { + Assert.True(HistorianServerVersionGate.IsValueGated(HistorianServiceInterface.History)); + Assert.True(HistorianServerVersionGate.IsValueGated(HistorianServiceInterface.Retrieval)); + Assert.True(HistorianServerVersionGate.IsValueGated(HistorianServiceInterface.Transaction)); + } + + [Fact] + public void ExpectedVersion_Status_Throws() + { + Assert.Throws( + () => HistorianServerVersionGate.ExpectedVersion(HistorianServiceInterface.Status)); + } + + [Fact] + public void Validate_NullOptions_Throws() + { + Assert.Throws( + () => HistorianServerVersionGate.Validate(HistorianServiceInterface.History, 11u, null!)); + } +} From fa9cde3e2f960a7d9f78991b3e1a2ea95853cb58 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 14:56:48 -0400 Subject: [PATCH 4/6] CW-1: reusable capture -> sanitize -> golden-fixture pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the highest-leverage reverse-engineering primitive from the roadmap: one path to turn a live operation buffer into a committable golden fixture. Unblocks every capture-tier item (R0.5, R1.x, R2.1). - ProtocolCaptureSanitizer: redacts identity-bearing values (host, tag, user, machine) from a native buffer in BOTH ASCII and UTF-16LE, overwriting in place with an 'X' fill so length and every field offset are preserved (keeps the fixture useful for byte-layout RE). ASCII-letter matching is case-insensitive; secrets < 3 chars are skipped to avoid collision corruption. AssertNoSecretsRemain is a fail-closed safety net that refuses to emit if any value survives. - ProtocolFixtureWriter: serializes a capture to fixtures/protocol//.json with sanitized hex, length, SHA-256 of the sanitized bytes, and a scrub report. Timestamps are passed in (deterministic / testable). - capture-tag-info CLI command: captures a live GetTagInfoFromName response and writes the fixture. The same native bytes ride inside 2023 R2 gRPC GetTagInfosFromName, so the fixture is transport-agnostic. - 11 unit tests for the sanitizer/writer (test project now references the RE tool). - First real fixture: get-tag-info/analog-*.json — a 98-byte Int4 CTagMetadata buffer captured live from the local Historian 2020 server, tag name redacted, verified to contain no identity (descriptor 03 c3 00 31 = Int4, as documented). 180 non-live unit tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../get-tag-info/analog-20260619185546.json | 18 ++ .../AVEVA.Historian.Client.Tests.csproj | 2 + .../ProtocolCaptureSanitizerTests.cs | 140 +++++++++++++++ .../Capture/ProtocolCaptureSanitizer.cs | 163 ++++++++++++++++++ .../Capture/ProtocolFixtureWriter.cs | 89 ++++++++++ .../Program.cs | 90 ++++++++++ 6 files changed, 502 insertions(+) create mode 100644 fixtures/protocol/get-tag-info/analog-20260619185546.json create mode 100644 tests/AVEVA.Historian.Client.Tests/ProtocolCaptureSanitizerTests.cs create mode 100644 tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolCaptureSanitizer.cs create mode 100644 tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolFixtureWriter.cs diff --git a/fixtures/protocol/get-tag-info/analog-20260619185546.json b/fixtures/protocol/get-tag-info/analog-20260619185546.json new file mode 100644 index 0000000..565ac45 --- /dev/null +++ b/fixtures/protocol/get-tag-info/analog-20260619185546.json @@ -0,0 +1,18 @@ +{ + "op": "get-tag-info", + "capturedUtc": "2026-06-19T18:55:46.5988258Z", + "notes": "RetrievalService.GetTagInfoFromName response (CTagMetadata buffer); identical bytes on 2023 R2 gRPC GetTagInfosFromName.", + "request": null, + "response": { + "length": 98, + "sha256": "cdda36baa869355b52ccb4be2735ccacfa2da69f0cafe62e88b807f1a05089fd", + "hex": "03c3003184228c4058e1874a984b3dbecbe0aa42ee000000091d0058585858585858585858585858585858585858585858585858585858580904004d44415302030102000000d057f49465d8dc010a0000000000000024400000000000002440fe00", + "redactions": [ + { + "secret": "tag", + "asciiMatches": 1, + "utf16Matches": 0 + } + ] + } +} \ No newline at end of file diff --git a/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj b/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj index 172b102..567ca3a 100644 --- a/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj +++ b/tests/AVEVA.Historian.Client.Tests/AVEVA.Historian.Client.Tests.csproj @@ -21,6 +21,8 @@ + + \ No newline at end of file diff --git a/tests/AVEVA.Historian.Client.Tests/ProtocolCaptureSanitizerTests.cs b/tests/AVEVA.Historian.Client.Tests/ProtocolCaptureSanitizerTests.cs new file mode 100644 index 0000000..14529bc --- /dev/null +++ b/tests/AVEVA.Historian.Client.Tests/ProtocolCaptureSanitizerTests.cs @@ -0,0 +1,140 @@ +using System.Text; +using System.Text.Json; +using AVEVA.Historian.ReverseEngineering.Capture; + +namespace AVEVA.Historian.Client.Tests; + +/// +/// Unit coverage for the CW-1 capture sanitizer and fixture writer — the reusable +/// "redact identity → emit committable fixture" core that all capture-tier work depends on. +/// +public sealed class ProtocolCaptureSanitizerTests +{ + private static byte[] Ascii(string s) => Encoding.ASCII.GetBytes(s); + + private static byte[] Utf16(string s) => Encoding.Unicode.GetBytes(s); + + [Fact] + public void Sanitize_RedactsAsciiOccurrence_PreservingLength() + { + byte[] buffer = [0x01, 0x02, .. Ascii("SECRETTAG"), 0x03]; + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("tag", "SECRETTAG")]); + + Assert.Equal(buffer.Length, result.Sanitized.Length); + Assert.Equal(0x01, result.Sanitized[0]); + Assert.Equal(0x03, result.Sanitized[^1]); + Assert.DoesNotContain(Ascii("SECRETTAG"), result.Sanitized); // value gone + Assert.Equal(1, result.Report[0].AsciiMatches); + Assert.Equal(0, result.Report[0].Utf16Matches); + } + + [Fact] + public void Sanitize_RedactsUtf16Occurrence() + { + byte[] buffer = [0xAA, .. Utf16("HostName"), 0xBB]; + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("host", "HostName")]); + + Assert.Equal(0, result.Report[0].AsciiMatches); + Assert.Equal(1, result.Report[0].Utf16Matches); + Assert.Equal(0xAA, result.Sanitized[0]); + Assert.Equal(0xBB, result.Sanitized[^1]); + } + + [Fact] + public void Sanitize_IsCaseInsensitiveForAsciiLetters() + { + byte[] buffer = Ascii("myserver01"); + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("host", "MyServer01")]); + + Assert.Equal(1, result.Report[0].AsciiMatches); + Assert.All(result.Sanitized, b => Assert.Equal(ProtocolCaptureSanitizer.FillByte, b)); + } + + [Fact] + public void Sanitize_RedactsMultipleOccurrences() + { + byte[] buffer = [.. Ascii("TagA"), 0x00, .. Ascii("TagA"), 0x00, .. Ascii("TagA")]; + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("tag", "TagA")]); + + Assert.Equal(3, result.Report[0].AsciiMatches); + Assert.Equal(3, result.TotalRedactions); + } + + [Fact] + public void Sanitize_IgnoresShortSecrets_ToAvoidCollisionCorruption() + { + byte[] buffer = [0x41, 0x42, 0x43]; // "ABC" + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("x", "AB")]); // length 2 < MinSecretLength + + Assert.Equal(buffer, result.Sanitized); // untouched + Assert.Equal(0, result.TotalRedactions); + } + + [Fact] + public void Sanitize_LeavesUnrelatedBytesUntouched() + { + byte[] buffer = [.. Ascii("keepme"), .. Ascii("DROPME"), .. Ascii("keepme")]; + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("s", "DROPME")]); + + Assert.Equal(Ascii("keepme"), result.Sanitized[..6]); + Assert.Equal(Ascii("keepme"), result.Sanitized[^6..]); + } + + [Fact] + public void AssertNoSecretsRemain_Passes_WhenRedacted() + { + byte[] buffer = Ascii("prefix-SECRET-suffix"); + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(buffer, [new CaptureSecret("s", "SECRET")]); + ProtocolCaptureSanitizer.AssertNoSecretsRemain(result.Sanitized, [new CaptureSecret("s", "SECRET")]); + } + + [Fact] + public void AssertNoSecretsRemain_Throws_WhenSecretSurvives() + { + byte[] buffer = Ascii("prefix-SECRET-suffix"); + Assert.Throws( + () => ProtocolCaptureSanitizer.AssertNoSecretsRemain(buffer, [new CaptureSecret("s", "SECRET")])); + } + + [Fact] + public void FixtureWriter_BuildJson_OmitsRawIdentity_AndRecordsScrubReport() + { + byte[] response = [0x4E, .. Utf16("CustomerTag.PV"), 0xFE, 0x00]; + var capture = new ProtocolCapture("get-tag-info", Request: null, Response: response, Notes: "live 2020 server"); + var secrets = new[] { new CaptureSecret("tag", "CustomerTag.PV") }; + + string json = ProtocolFixtureWriter.BuildFixtureJson(capture, secrets, "2026-06-19T00:00:00Z"); + + Assert.DoesNotContain("CustomerTag", json); // identity scrubbed from hex + using JsonDocument doc = JsonDocument.Parse(json); + JsonElement root = doc.RootElement; + Assert.Equal("get-tag-info", root.GetProperty("op").GetString()); + Assert.Equal("2026-06-19T00:00:00Z", root.GetProperty("capturedUtc").GetString()); + Assert.Equal(JsonValueKind.Null, root.GetProperty("request").ValueKind); + JsonElement resp = root.GetProperty("response"); + Assert.Equal(response.Length, resp.GetProperty("length").GetInt32()); + Assert.Equal(64, resp.GetProperty("sha256").GetString()!.Length); + Assert.Equal("tag", resp.GetProperty("redactions")[0].GetProperty("secret").GetString()); + } + + [Fact] + public void FixtureWriter_Write_CreatesOpSubdirectoryFile() + { + string root = Path.Combine(Path.GetTempPath(), "histsdk-fixture-test-" + Guid.NewGuid().ToString("N")); + try + { + var capture = new ProtocolCapture("get-tag-info", Request: null, Response: [0x01, 0x02, 0x03], Notes: null); + string path = ProtocolFixtureWriter.Write(root, "sample", capture, [], "2026-06-19T00:00:00Z"); + + Assert.True(File.Exists(path)); + Assert.EndsWith(Path.Combine("get-tag-info", "sample.json"), path); + } + finally + { + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + } + } +} diff --git a/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolCaptureSanitizer.cs b/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolCaptureSanitizer.cs new file mode 100644 index 0000000..17348f3 --- /dev/null +++ b/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolCaptureSanitizer.cs @@ -0,0 +1,163 @@ +using System.Text; + +namespace AVEVA.Historian.ReverseEngineering.Capture; + +/// A sensitive value to scrub from a captured buffer before it can be committed. +/// Stable label (e.g. "host", "tag", "user") recorded in the scrub report. +/// The literal value to redact wherever it appears in the buffer. +public sealed record CaptureSecret(string Name, string Value); + +/// How many times a secret was found and redacted, per encoding. +public sealed record ScrubCount(string Name, int AsciiMatches, int Utf16Matches) +{ + public int Total => AsciiMatches + Utf16Matches; +} + +/// Result of sanitizing a captured buffer: the redacted copy plus a per-secret report. +public sealed record SanitizeResult(byte[] Sanitized, IReadOnlyList Report) +{ + public int TotalRedactions + { + get + { + int total = 0; + foreach (ScrubCount count in Report) + { + total += count.Total; + } + + return total; + } + } +} + +/// +/// CW-1 core: redacts identity-bearing values (hostnames, tag names, user names) from a captured +/// native Historian buffer so the result can be saved as a committable golden fixture. +/// +/// Each secret is matched in both ASCII/UTF-8 and UTF-16LE (the two encodings AVEVA's +/// native buffers use for embedded strings) and overwritten in place with a fixed fill byte. The +/// redaction preserves the buffer's exact length and every field offset, so the sanitized fixture +/// remains useful for byte-layout reverse engineering while carrying none of the original identity. +/// +/// ASCII-letter matching is case-insensitive (servers may echo a tag/host in a different case than +/// requested); other bytes match exactly. Secrets shorter than are +/// ignored to avoid corrupting unrelated bytes that coincidentally collide with a short value. +/// +public static class ProtocolCaptureSanitizer +{ + /// Fill byte written over a redacted region ('X'). Chosen to be obviously non-data on inspection. + public const byte FillByte = (byte)'X'; + + /// Secrets shorter than this many characters are not scrubbed (too collision-prone). + public const int MinSecretLength = 3; + + public static SanitizeResult Sanitize(ReadOnlySpan buffer, IReadOnlyList secrets) + { + ArgumentNullException.ThrowIfNull(secrets); + + byte[] working = buffer.ToArray(); + List report = new(secrets.Count); + + foreach (CaptureSecret secret in secrets) + { + if (string.IsNullOrEmpty(secret.Value) || secret.Value.Length < MinSecretLength) + { + report.Add(new ScrubCount(secret.Name, 0, 0)); + continue; + } + + int ascii = RedactPattern(working, Encoding.ASCII.GetBytes(secret.Value)); + int utf16 = RedactPattern(working, Encoding.Unicode.GetBytes(secret.Value)); + report.Add(new ScrubCount(secret.Name, ascii, utf16)); + } + + return new SanitizeResult(working, report); + } + + /// + /// Safety net: throws if any secret value still survives (in either encoding) in the buffer. + /// Call after before writing a fixture so a redaction gap can never + /// leak identity into a committed file. + /// + public static void AssertNoSecretsRemain(ReadOnlySpan sanitized, IReadOnlyList secrets) + { + ArgumentNullException.ThrowIfNull(secrets); + + foreach (CaptureSecret secret in secrets) + { + if (string.IsNullOrEmpty(secret.Value) || secret.Value.Length < MinSecretLength) + { + continue; + } + + if (IndexOf(sanitized, Encoding.ASCII.GetBytes(secret.Value), 0) >= 0 + || IndexOf(sanitized, Encoding.Unicode.GetBytes(secret.Value), 0) >= 0) + { + throw new InvalidOperationException( + $"Sanitized buffer still contains secret '{secret.Name}'. Refusing to emit an unsanitized fixture."); + } + } + } + + private static int RedactPattern(byte[] buffer, byte[] pattern) + { + if (pattern.Length == 0) + { + return 0; + } + + int matches = 0; + int index = 0; + while ((index = IndexOf(buffer, pattern, index)) >= 0) + { + buffer.AsSpan(index, pattern.Length).Fill(FillByte); + index += pattern.Length; + matches++; + } + + return matches; + } + + private static int IndexOf(ReadOnlySpan haystack, ReadOnlySpan needle, int start) + { + if (needle.Length == 0 || haystack.Length - start < needle.Length) + { + return -1; + } + + for (int i = start; i <= haystack.Length - needle.Length; i++) + { + bool match = true; + for (int j = 0; j < needle.Length; j++) + { + if (!BytesEqualCaseInsensitive(haystack[i + j], needle[j])) + { + match = false; + break; + } + } + + if (match) + { + return i; + } + } + + return -1; + } + + /// Compare bytes, treating ASCII letters case-insensitively; all other bytes exactly. + private static bool BytesEqualCaseInsensitive(byte a, byte b) + { + if (a == b) + { + return true; + } + + return ToLowerAscii(a) == ToLowerAscii(b); + } + + private static byte ToLowerAscii(byte value) => + value is >= (byte)'A' and <= (byte)'Z' ? (byte)(value + 32) : value; +} diff --git a/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolFixtureWriter.cs b/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolFixtureWriter.cs new file mode 100644 index 0000000..69f356c --- /dev/null +++ b/tools/AVEVA.Historian.ReverseEngineering/Capture/ProtocolFixtureWriter.cs @@ -0,0 +1,89 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace AVEVA.Historian.ReverseEngineering.Capture; + +/// One captured operation: the (optional) request buffer and the response buffer, raw. +public sealed record ProtocolCapture(string Op, byte[]? Request, byte[]? Response, string? Notes = null); + +/// +/// CW-1 fixture writer: takes a live , redacts it with +/// , and writes a committable JSON fixture under +/// fixtures/protocol/<op>/. The fixture records sanitized hex, lengths, SHA-256 of the +/// sanitized bytes, and the scrub report — never the original identity-bearing bytes. +/// +/// Timestamps are passed in (never generated here) so the writer stays deterministic and testable. +/// +public static class ProtocolFixtureWriter +{ + public static string BuildFixtureJson( + ProtocolCapture capture, + IReadOnlyList secrets, + string capturedUtcIso) + { + ArgumentNullException.ThrowIfNull(capture); + + BufferSection? request = BuildSection(capture.Request, secrets); + BufferSection? response = BuildSection(capture.Response, secrets); + + var document = new + { + op = capture.Op, + capturedUtc = capturedUtcIso, + notes = capture.Notes, + request, + response, + }; + + return JsonSerializer.Serialize(document, new JsonSerializerOptions + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }); + } + + /// Serializes the fixture and writes it to /<op>/<name>.json. Returns the path. + public static string Write( + string fixtureRoot, + string name, + ProtocolCapture capture, + IReadOnlyList secrets, + string capturedUtcIso) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fixtureRoot); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentNullException.ThrowIfNull(capture); + + string json = BuildFixtureJson(capture, secrets, capturedUtcIso); + string directory = Path.Combine(fixtureRoot, capture.Op); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, name + ".json"); + File.WriteAllText(path, json, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + return path; + } + + private static BufferSection? BuildSection(byte[]? raw, IReadOnlyList secrets) + { + if (raw is null) + { + return null; + } + + SanitizeResult result = ProtocolCaptureSanitizer.Sanitize(raw, secrets); + ProtocolCaptureSanitizer.AssertNoSecretsRemain(result.Sanitized, secrets); + + return new BufferSection( + Length: raw.Length, + Sha256: Convert.ToHexString(SHA256.HashData(result.Sanitized)).ToLowerInvariant(), + Hex: Convert.ToHexString(result.Sanitized).ToLowerInvariant(), + Redactions: result.Report + .Where(r => r.Total > 0) + .Select(r => new RedactionEntry(r.Name, r.AsciiMatches, r.Utf16Matches)) + .ToArray()); + } + + private sealed record BufferSection(int Length, string Sha256, string Hex, IReadOnlyList Redactions); + + private sealed record RedactionEntry(string Secret, int AsciiMatches, int Utf16Matches); +} diff --git a/tools/AVEVA.Historian.ReverseEngineering/Program.cs b/tools/AVEVA.Historian.ReverseEngineering/Program.cs index b70d1eb..014cd6b 100644 --- a/tools/AVEVA.Historian.ReverseEngineering/Program.cs +++ b/tools/AVEVA.Historian.ReverseEngineering/Program.cs @@ -12,8 +12,10 @@ using System.Security.Cryptography; using System.Runtime.Versioning; using System.Text; using System.Text.Json; +using AVEVA.Historian.Client; using AVEVA.Historian.Client.Wcf; using AVEVA.Historian.Client.Wcf.Contracts; +using AVEVA.Historian.ReverseEngineering.Capture; using dnlib.DotNet; using dnlib.DotNet.Emit; @@ -68,6 +70,7 @@ try "wcf-start-event-query" => StartWcfEventQuery(args), "wcf-register-event-tag" => RegisterEventTagAndStartQuery(args), "wcf-add-event-tag" => AddEventTagAndStartQuery(args), + "capture-tag-info" => CaptureTagInfo(args), _ => UnknownCommand(args[0]) }; } @@ -3605,6 +3608,90 @@ static int ProbeWcfTagInfo(string[] args) return result.Success ? 0 : 1; } +// CW-1: capture a live GetTagInfoFromName response buffer and persist it as a sanitized, +// committable golden fixture under fixtures/protocol/get-tag-info/. The same native byte blob +// travels inside the 2023 R2 gRPC RetrievalService.GetTagInfosFromName response, so the fixture +// is transport-agnostic. Usage: capture-tag-info [host] [port] [tag] [fixture-root] +static int CaptureTagInfo(string[] args) +{ + string host = args.Length > 1 ? args[1] : "localhost"; + int port = args.Length > 2 && int.TryParse(args[2], out int parsedPort) + ? parsedPort + : HistorianWcfBindingFactory.DefaultPort; + string tag = args.Length > 3 ? args[3] : "OtOpcUaParityTest_001.Counter"; + string fixtureRoot = args.Length > 4 ? args[4] : ResolveFixtureRoot(); + + var options = new HistorianClientOptions + { + Host = host, + Port = port, + IntegratedSecurity = true, + }; + + IReadOnlyDictionary raw = HistorianWcfTagClient.GetTagInfoRawBytesForProbe(options, [tag]); + byte[]? response = raw.TryGetValue(tag, out byte[]? bytes) ? bytes : null; + if (response is null || response.Length == 0) + { + Console.Error.WriteLine($"GetTagInfoFromName returned no bytes for the requested tag against {host}:{port}."); + return 1; + } + + // Redact every identity-bearing value that could appear in the buffer: the requested tag, + // the host/machine name, and the captured user. The sanitizer scrubs ASCII + UTF-16LE and + // refuses to emit if any value survives. + var secrets = new List + { + new("tag", tag), + new("host", host), + new("machine", Environment.MachineName), + new("user", Environment.UserName), + }; + string? envUser = Environment.GetEnvironmentVariable("HISTORIAN_USER"); + if (!string.IsNullOrWhiteSpace(envUser)) + { + secrets.Add(new CaptureSecret("env-user", envUser)); + } + + var capture = new ProtocolCapture( + Op: "get-tag-info", + Request: null, + Response: response, + Notes: "RetrievalService.GetTagInfoFromName response (CTagMetadata buffer); identical bytes on 2023 R2 gRPC GetTagInfosFromName."); + + string capturedUtc = DateTime.UtcNow.ToString("o"); + string path = ProtocolFixtureWriter.Write(fixtureRoot, $"analog-{DateTime.UtcNow:yyyyMMddHHmmss}", capture, secrets, capturedUtc); + + var summary = new + { + Op = capture.Op, + ResponseLength = response.Length, + FixturePath = path, + Redactions = ProtocolCaptureSanitizer.Sanitize(response, secrets).Report + .Where(r => r.Total > 0) + .Select(r => new { r.Name, r.AsciiMatches, r.Utf16Matches }), + }; + Console.WriteLine(JsonSerializer.Serialize(summary, CreateJsonOptions())); + return 0; +} + +// Walk up from the working directory to the repo root (the directory holding Histsdk.slnx) and +// return its fixtures/protocol path; fall back to fixtures/protocol under the CWD. +static string ResolveFixtureRoot() +{ + DirectoryInfo? dir = new(Directory.GetCurrentDirectory()); + while (dir is not null) + { + if (File.Exists(Path.Combine(dir.FullName, "Histsdk.slnx"))) + { + return Path.Combine(dir.FullName, "fixtures", "protocol"); + } + + dir = dir.Parent; + } + + return Path.Combine(Directory.GetCurrentDirectory(), "fixtures", "protocol"); +} + static int ProbeWcfLikeTagBrowse(string[] args) { string host = args.Length > 1 ? args[1] : "localhost"; @@ -6370,6 +6457,9 @@ static void PrintHelp() instrument-tagquery-gettaginfo [dll-path] [output-path] Write a reverse-only wrapper copy that logs TagQuery CTagMetadata vectors. mark Emit a timestamp marker for Wireshark/API Monitor notes. + capture-tag-info [host] [port] [tag] [fixture-root] + CW-1: capture a live GetTagInfoFromName buffer and write a + sanitized golden fixture to fixtures/protocol/get-tag-info/. wcf-probe [host] [port] Probe Hist/Retr/Stat WCF GetV endpoints with MDAS encoding. wcf-cert-probe [host] [port] [dns] Probe HistCert GetV with MDAS over TLS transport security. From cf5a66e0467192225695782c333a782f4ed4c3dd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 14:57:38 -0400 Subject: [PATCH 5/6] docs/plans: mark R0.6 + CW-1 done; note 2020-only live-verification constraint The local Historian is 2020 (WCF/32568); the 2023 R2 gRPC endpoint (32565) is absent, so M0 gRPC routing can be unit-tested but not live-verified here. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/plans/hcal-roadmap.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/plans/hcal-roadmap.md b/docs/plans/hcal-roadmap.md index ac7ed8d..53432b3 100644 --- a/docs/plans/hcal-roadmap.md +++ b/docs/plans/hcal-roadmap.md @@ -9,6 +9,21 @@ HCAL replacement, built on the **2023 R2 gRPC transport**. Derived from > protocol serializer/parser + golden-byte unit test + an env-gated live integration > test against the local Historian. +## Progress (updated 2026-06-19) + +- ✅ **R0.6 version gate** — `HistorianServerVersionGate` + `HistorianClientOptions.VerifyServerInterfaceVersion`; + fail-closed on connect, wired into both WCF and gRPC paths. Supported versions are + evidence-based (Hist=11, Retr=4, Trx=2; Status reachability-only), captured from the + live server. 10 unit tests. +- ✅ **CW-1 capture pipeline** — `ProtocolCaptureSanitizer` + `ProtocolFixtureWriter` + + `capture-tag-info` CLI command; produces sanitized `fixtures/protocol//` golden files. + 11 unit tests. First fixture: `get-tag-info/analog-*.json`. + +> ⚠️ **Live-verification constraint:** the local Historian is **2020** (WCF, port 32568) — the +> 2023 R2 gRPC endpoint (32565) is absent. M0's gRPC routing (R0.1–R0.4) can be built and +> golden-byte/unit-tested here but **cannot be live-verified** without an actual 2023 R2 server. +> Treat gRPC ops as unverified until then; the byte payloads remain the proven 2020 protocol. + ## Guiding principles 1. **gRPC-first.** New ops go on the `RemoteGrpc` transport (clean protobuf envelope); From 7d5aeaeb0647e2b24515ce0f9bae41bf9bc3cb3d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 19 Jun 2026 15:12:41 -0400 Subject: [PATCH 6/6] Strengthen live event-read test: assert well-formed parsed events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadEventsAsync verified to return real, parsed events against the local 2020 server (e.g. User.Write with 18 properties) — the row parser (HistorianEventRowProtocol v9) is wired and works. The prior test only asserted NotNull with a stale "row format not yet decoded" comment. - Renamed to ReadEventsAsync_AgainstLocalHistorian_ReturnsWellFormedEvents. - Widened the window to 30 days (robust against a quiet recent window). - Asserts NotEmpty + per-event well-formedness (non-empty Type, non-null Properties, EventTimeUtc within the queried window) — matching the ReadRawAsync test's NotEmpty style. - Documents the known limitation: enumeration stops at the first benign `type=4 code=85` soft-terminal, so this verifies parsing correctness rather than exhaustive retrieval (draining all rows needs the code-85 decode, a capture task). Passes live (1 event over 30 days). Non-live unit suite unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../HistorianClientIntegrationTests.cs | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs index f906675..be7e6ac 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianClientIntegrationTests.cs @@ -211,7 +211,7 @@ public sealed class HistorianClientIntegrationTests } [Fact] - public async Task ReadEventsAsync_AgainstLocalHistorian_DoesNotThrow() + public async Task ReadEventsAsync_AgainstLocalHistorian_ReturnsWellFormedEvents() { string? host = Environment.GetEnvironmentVariable("HISTORIAN_HOST"); if (string.IsNullOrWhiteSpace(host) || !string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase) || !OperatingSystem.IsWindows()) @@ -227,18 +227,28 @@ public sealed class HistorianClientIntegrationTests }); DateTime endUtc = DateTime.UtcNow; - DateTime startUtc = endUtc - TimeSpan.FromDays(7); + DateTime startUtc = endUtc - TimeSpan.FromDays(30); - // The event-row WCF wire format is not yet decoded; this test verifies the chain - // (ValCl + Open2 + Retr.IsOriginalAllowed + Retr.StartEventQuery) reaches the server - // without throwing. An empty event list is acceptable until row parsing is wired. + // The full chain (ValCl + Open2 + Retr.IsOriginalAllowed + Retr.StartEventQuery + + // GetNextEventQueryResultBuffer + HistorianEventRowProtocol.Parse) returns real, parsed + // events. Requires the local store to hold events in the window — System-Platform + // alarm/user-write events are present on a working Historian. NOTE: enumeration currently + // stops at the first benign `type=4 code=85` soft-terminal, so this verifies parsing + // correctness rather than exhaustive retrieval (decoding code 85 to drain all rows is a + // separate capture task). List events = []; await foreach (AVEVA.Historian.Client.Models.HistorianEvent evt in client.ReadEventsAsync(startUtc, endUtc, CancellationToken.None)) { events.Add(evt); } - Assert.NotNull(events); + Assert.NotEmpty(events); + Assert.All(events, evt => + { + Assert.False(string.IsNullOrWhiteSpace(evt.Type)); // e.g. "User.Write", "Alarm.Set" + Assert.NotNull(evt.Properties); + Assert.InRange(evt.EventTimeUtc, startUtc - TimeSpan.FromDays(1), endUtc + TimeSpan.FromDays(1)); + }); } [Fact]