diff --git a/CLAUDE.md b/CLAUDE.md index 869be19..6742561 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ Writes (added 2026-05-04 by explicit user request — do not extend further with - `EnsureTagAsync` for analog types: Float, Double, Int2, Int4, UInt4 (live-verified end-to-end). Other types (SingleByteString/DoubleByteString/Int1/Int8/UInt8) fail at native AddTag — likely require a different path and are intentionally not supported. `MinEU`/`MaxEU`/`MinRaw`/`MaxRaw` all round-trip into the DB. By default `ApplyScaling=false` and the server mirrors MinRaw→MinEU and sets `AnalogTag.Scaling=0`; set `ApplyScaling=true` on the definition to persist distinct raw bounds with `AnalogTag.Scaling=1`. The wire encoding is the trailer's second byte (`FE 00` vs `FE 01`). - `DeleteTagAsync` -- `AddHistoricalValuesAsync` (added 2026-06-21 by explicit user request — M3 historical/backfill writes). **gRPC-only** (`HistorianTransport.RemoteGrpc`); non-gRPC transports throw `ProtocolEvidenceMissingException`. Reverse-engineered by capturing the native 2023 R2 client: the historical write rides `HistoryService.AddStreamValues` with an "ON" storage-sample buffer (`HistorianHistoricalWriteProtocol`, golden-tested), NOT the TransactionService `AddNonStreamValues` path the static decompile suggested. Orchestrator (`HistorianGrpcHistoricalWriteOrchestrator`): write-enabled session → `GetTagInfosFromName` (resolves the per-tag GUID = the tag-info `TypeId`) → `AddStreamValues`. Tag must pre-exist (`EnsureTagAsync`). Float value encoding only (the captured type; value = `u32(0) + float32` in an 8-byte slot). Live-validated end-to-end (write + read-back) against the 2023 R2 server. The D2/`AddS2` cache gate (err 129) does NOT block the primed 2023 R2 client. See `docs/plans/revision-write-path.md` §"R3.1 CAPTURED". +- `AddHistoricalValuesAsync` (added 2026-06-21 by explicit user request — M3 historical/backfill writes). **gRPC-only** (`HistorianTransport.RemoteGrpc`); non-gRPC transports throw `ProtocolEvidenceMissingException`. Reverse-engineered by capturing the native 2023 R2 client: the historical write rides `HistoryService.AddStreamValues` with an "ON" storage-sample buffer (`HistorianHistoricalWriteProtocol`, golden-tested), NOT the TransactionService `AddNonStreamValues` path the static decompile suggested. Orchestrator (`HistorianGrpcHistoricalWriteOrchestrator`): write-enabled session → `GetTagInfosFromName` (resolves the per-tag GUID = the tag-info `TypeId`, and maps the data type via `MapDataType`) → `AddStreamValues`. Tag must pre-exist (`EnsureTagAsync`). Supports all five analog types `EnsureTagAsync` does — **Float, Double, Int2, Int4, UInt4** (all captured live + golden-tested + write/read-back validated). The 4-byte value descriptor is constant (`C0 10 01 00`); the value is `u32(0) + native-width value` (float32 / double64 / int16 / int32 / uint32) selected by the tag's declared type. Other tag types throw `ProtocolEvidenceMissingException`. Live-validated end-to-end against the 2023 R2 server. The D2/`AddS2` cache gate (err 129) does NOT block the primed 2023 R2 client. See `docs/plans/revision-write-path.md` §"R3.1 CAPTURED". `AddS2` (streaming process-sample writes for user tags) remains architecturally blocked — the server cache only ingests from configured IOServers/ApplicationServer pipelines. Do not add streaming write-samples support. (`AddHistoricalValuesAsync` is the distinct *non-streamed original/backfill* path and is supported.) diff --git a/docs/plans/revision-write-path.md b/docs/plans/revision-write-path.md index 089104e..a5f1e5c 100644 --- a/docs/plans/revision-write-path.md +++ b/docs/plans/revision-write-path.md @@ -249,9 +249,11 @@ over `RemoteGrpc`: `HistorianGrpcHistoricalWriteOrchestrator` opens a write-enab `GetTagInfosFromName` (resolves the per-tag GUID = the tag-info record's `TypeId`) → `HistoryService.AddStreamValues` ("ON" buffer from `HistorianHistoricalWriteProtocol`, golden-tested) per sample. The pure-managed SDK wrote a value and read it back live (gated test -`AddHistoricalValuesAsync_OverGrpc_WritesAndReadsBack`). Float value encoding only (the captured type); -gRPC-only. Capture artifacts (gitignored): -`artifacts/reverse-engineering/grpc-nonstream-capture/captureB4.ndjson`. +`AddHistoricalValuesAsync_OverGrpc_WritesAndReadsBack`). **All five analog types captured + validated** +(Float/Double/Int2/Int4/UInt4): the 4-byte value descriptor `C0 10 01 00` is **constant across types**; +the value is `u32(0) + native-width value` (float32 / double64 / int16 / int32 / uint32) selected by the +tag's declared type (the orchestrator maps it from the tag-info `NativeDataTypeDescriptor`). gRPC-only. +Capture artifacts (gitignored): `artifacts/reverse-engineering/grpc-nonstream-capture/cap-*.ndjson`. --- diff --git a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHistoricalWriteOrchestrator.cs b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHistoricalWriteOrchestrator.cs index 8f033ba..effc7c4 100644 --- a/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHistoricalWriteOrchestrator.cs +++ b/src/AVEVA.Historian.Client/Grpc/HistorianGrpcHistoricalWriteOrchestrator.cs @@ -77,6 +77,7 @@ internal sealed class HistorianGrpcHistoricalWriteOrchestrator } Guid tagGuid = parsed[0].TypeId; + HistorianDataType dataType = HistorianWcfTagClient.MapDataType(parsed[0].NativeDataTypeDescriptor); var historyClient = new GrpcHistory.HistoryService.HistoryServiceClient(connection.Channel); foreach (HistorianHistoricalValue value in values) @@ -84,6 +85,7 @@ internal sealed class HistorianGrpcHistoricalWriteOrchestrator cancellationToken.ThrowIfCancellationRequested(); byte[] buffer = HistorianHistoricalWriteProtocol.SerializeAddStreamValuesBuffer( tagGuid, + dataType, value.TimestampUtc, value.Value, DateTime.UtcNow, diff --git a/src/AVEVA.Historian.Client/Wcf/HistorianHistoricalWriteProtocol.cs b/src/AVEVA.Historian.Client/Wcf/HistorianHistoricalWriteProtocol.cs index a5c223d..f485d0c 100644 --- a/src/AVEVA.Historian.Client/Wcf/HistorianHistoricalWriteProtocol.cs +++ b/src/AVEVA.Historian.Client/Wcf/HistorianHistoricalWriteProtocol.cs @@ -1,14 +1,14 @@ using System.Buffers.Binary; +using AVEVA.Historian.Client.Models; namespace AVEVA.Historian.Client.Wcf; /// /// Serializer for the M3 historical (non-streamed original / backfill) value write — the -/// HistoryService.AddStreamValues values buffer. Decoded byte-for-byte from a live -/// capture of the native 2023 R2 gRPC client driving -/// HistorianAccess.AddNonStreamedValue → SendValues against a sandbox tag (see -/// docs/plans/revision-write-path.md §"R3.1 CAPTURED"). The native non-streamed write does -/// NOT use the TransactionService AddNonStreamValues path the static decompile suggested — it +/// HistoryService.AddStreamValues values buffer. Decoded byte-for-byte from live +/// captures of the native 2023 R2 gRPC client driving +/// HistorianAccess.AddNonStreamedValue → SendValues against sandbox tags of each analog type +/// (see docs/plans/revision-write-path.md §"R3.1 CAPTURED"). The native non-streamed write /// rides HistoryService.AddStreamValues with an "ON" storage-sample buffer, the analog sibling /// of the AddS2 "OS" event buffer (). /// @@ -18,46 +18,61 @@ namespace AVEVA.Historian.Client.Wcf; /// 0x02 UInt16 sampleCount = 1 /// 0x04 UInt32 10 + valueBlob.Length // total buffer length /// 0x08 UInt16 valueBlob.Length -/// 0x0A valueBlob (46 bytes for one analog double): +/// 0x0A valueBlob: /// +0x00 GUID tag GUID (the per-tag GUID = the value ParseTagInfoRecord reads as "typeId") /// +0x10 Int64 sample FILETIME (UTC — the VTQ value timestamp) /// +0x18 UInt16 OpcQuality = 192 (good) -/// +0x1A 4 bytes analog value descriptor (constant for the Float path: C0 10 01 00) +/// +0x1A 4 bytes value descriptor — CONSTANT C0 10 01 00 across all analog types (the server +/// interprets the value bytes by the tag's declared type) /// +0x1E Int64 received/version FILETIME (UTC) -/// +0x26 UInt32 0 // value high dword (zero for a 4-byte Float) -/// +0x2A Float32 value // the Float value sits in the high half of the 8-byte slot +/// +0x26 UInt32 0 // value high dword (constant zero) +/// +0x2A value bytes, native width by tag type: +/// Float → Float32(4) · Double → Float64(8) · Int2 → Int16(2) +/// Int4 → Int32(4) · UInt4 → UInt32(4) /// /// -/// Captured against a Float tag: the value occupies an 8-byte slot as u32(0) + float32(value) -/// (the 4-byte IEEE-754 float in the high dword, NOT an 8-byte double). Only the Float encoding is -/// captured; Double/Int/string tag types use a different descriptor + value width and are rejected -/// until captured. +/// Captured for the five analog types EnsureTagAsync supports (Float/Double/Int2/Int4/UInt4). +/// Other tag types have no captured value encoding and are rejected. /// internal static class HistorianHistoricalWriteProtocol { public const ushort BufferSignature = 0x4E4F; // "ON" public const ushort OpcQualityGood = 192; - // Captured constant for the analog double value path. The other observed quality/descriptor - // bytes in the AddS2 "OS" buffer are event-specific; here this 4-byte block sits between the - // quality and the received-time FILETIME and was constant across captured analog writes. - private static readonly byte[] AnalogDoubleDescriptor = [0xC0, 0x10, 0x01, 0x00]; + // Captured constant: the 4-byte value descriptor was identical across all analog types + // (Float/Double/Int2/Int4/UInt4). The value type is conveyed by the tag's registration, not here. + private static readonly byte[] ValueDescriptor = [0xC0, 0x10, 0x01, 0x00]; - private const int ValueBlobLength = 16 + 8 + 2 + 4 + 8 + 8; // 46 + // valueBlob fixed prefix: GUID(16) + sampleFT(8) + quality(2) + descriptor(4) + receivedFT(8) + + // value high dword(4) = 42, then the native-width value bytes. + private const int ValueBlobPrefixLength = 16 + 8 + 2 + 4 + 8 + 4; // 42 /// /// Builds the AddStreamValues values buffer for a single analog historical sample. - /// is the per-tag GUID (from the gRPC tag-info read), and + /// is the per-tag GUID (from the gRPC tag-info read), + /// is the tag's declared analog type (selects the value width), and /// is the storage/received timestamp the orchestrator stamps. + /// Throws for tag types without a captured encoding. /// public static byte[] SerializeAddStreamValuesBuffer( Guid tagGuid, + HistorianDataType dataType, DateTime sampleTimeUtc, double value, DateTime receivedTimeUtc, ushort quality = OpcQualityGood) { - byte[] valueBlob = SerializeValueBlob(tagGuid, sampleTimeUtc, value, receivedTimeUtc, quality); + byte[] valueBytes = EncodeNativeValue(dataType, value); + + byte[] valueBlob = new byte[ValueBlobPrefixLength + valueBytes.Length]; + Span span = valueBlob; + tagGuid.ToByteArray().CopyTo(span[..16]); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(16, 8), ToFileTime(sampleTimeUtc)); + BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(24, 2), quality); + ValueDescriptor.CopyTo(span.Slice(26, 4)); + BinaryPrimitives.WriteInt64LittleEndian(span.Slice(30, 8), ToFileTime(receivedTimeUtc)); + BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(38, 4), 0); // value high dword (constant 0) + valueBytes.CopyTo(span.Slice(42, valueBytes.Length)); byte[] buffer = new byte[10 + valueBlob.Length]; BinaryPrimitives.WriteUInt16LittleEndian(buffer.AsSpan(0, 2), BufferSignature); @@ -68,26 +83,45 @@ internal static class HistorianHistoricalWriteProtocol return buffer; } - private static byte[] SerializeValueBlob( - Guid tagGuid, - DateTime sampleTimeUtc, - double value, - DateTime receivedTimeUtc, - ushort quality) + private static byte[] EncodeNativeValue(HistorianDataType dataType, double value) { - byte[] blob = new byte[ValueBlobLength]; - Span span = blob; - - tagGuid.ToByteArray().CopyTo(span[..16]); - BinaryPrimitives.WriteInt64LittleEndian(span.Slice(16, 8), ToFileTime(sampleTimeUtc)); - BinaryPrimitives.WriteUInt16LittleEndian(span.Slice(24, 2), quality); - AnalogDoubleDescriptor.CopyTo(span.Slice(26, 4)); - BinaryPrimitives.WriteInt64LittleEndian(span.Slice(30, 8), ToFileTime(receivedTimeUtc)); - // The value sits in an 8-byte slot as u32(0) + float32(value): the captured Float tag stored - // a 4-byte IEEE-754 float in the high dword, not an 8-byte double. - BinaryPrimitives.WriteUInt32LittleEndian(span.Slice(38, 4), 0); - BinaryPrimitives.WriteSingleLittleEndian(span.Slice(42, 4), (float)value); - return blob; + switch (dataType) + { + case HistorianDataType.Float: + { + byte[] b = new byte[4]; + BinaryPrimitives.WriteSingleLittleEndian(b, (float)value); + return b; + } + case HistorianDataType.Double: + { + byte[] b = new byte[8]; + BinaryPrimitives.WriteDoubleLittleEndian(b, value); + return b; + } + case HistorianDataType.Int2: + { + byte[] b = new byte[2]; + BinaryPrimitives.WriteInt16LittleEndian(b, checked((short)value)); + return b; + } + case HistorianDataType.Int4: + { + byte[] b = new byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(b, checked((int)value)); + return b; + } + case HistorianDataType.UInt4: + { + byte[] b = new byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(b, checked((uint)value)); + return b; + } + default: + throw new ProtocolEvidenceMissingException( + $"AddHistoricalValuesAsync has no captured value encoding for tag data type '{dataType}'. " + + "Captured types: Float, Double, Int2, Int4, UInt4."); + } } private static long ToFileTime(DateTime value) diff --git a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs index 17043b5..6512464 100644 --- a/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/HistorianGrpcIntegrationTests.cs @@ -189,9 +189,10 @@ public sealed class HistorianGrpcIntegrationTests HistorianClient client = new(BuildOptions(host)); - // A backfill sample at a fixed historical second, with a distinctive value. + // A backfill sample at a fixed historical second, with a distinctive whole-number value so + // it round-trips for any analog tag type (Float/Double/Int2/Int4/UInt4). DateTime stamp = new DateTime(DateTime.UtcNow.Year, 1, 2, 3, 4, 5, DateTimeKind.Utc); - const double expected = 222.5; + const double expected = 7777; bool wrote = await client.AddHistoricalValuesAsync( sandboxTag!, [new HistorianHistoricalValue(stamp, expected)], diff --git a/tests/AVEVA.Historian.Client.Tests/WcfHistoricalWriteProtocolTests.cs b/tests/AVEVA.Historian.Client.Tests/WcfHistoricalWriteProtocolTests.cs index 20ac4a4..3e142d5 100644 --- a/tests/AVEVA.Historian.Client.Tests/WcfHistoricalWriteProtocolTests.cs +++ b/tests/AVEVA.Historian.Client.Tests/WcfHistoricalWriteProtocolTests.cs @@ -1,41 +1,60 @@ +using System.Buffers.Binary; +using AVEVA.Historian.Client.Models; using AVEVA.Historian.Client.Wcf; namespace AVEVA.Historian.Client.Tests; public sealed class WcfHistoricalWriteProtocolTests { - // The exact 56-byte HistoryService.AddStreamValues "values" buffer captured live from the native - // 2023 R2 client writing a historical Float sample (124.5) to a sandbox tag — see - // docs/plans/revision-write-path.md §"R3.1 CAPTURED". - private const string CapturedBufferHex = - "4f4e0100380000002e00" + // "ON" + count(1) + totalLen(56) + payloadLen(46) - "d51d3107e9f8664793b155d3d1aef544" + // tag GUID 07311dd5-f8e9-4766-93b1-55d3d1aef544 - "70df38b1d101dd01" + // sample FILETIME - "c000" + // OpcQuality = 192 - "c0100100" + // analog double descriptor - "e64bcb74e201dd01" + // received/version FILETIME - "000000000000f942"; // double 124.5 - - [Fact] - public void SerializeAddStreamValuesBuffer_MatchesCapturedNativeBuffer() + // Exact HistoryService.AddStreamValues "values" buffers captured live from the native 2023 R2 + // client writing one historical sample to a sandbox tag of each analog type — see + // docs/plans/revision-write-path.md §"R3.1 CAPTURED". The value descriptor (C0 10 01 00) is + // constant across types; only the value width differs. + [Theory] + [InlineData(HistorianDataType.Float, 124.5, + "4f4e0100380000002e00d51d3107e9f8664793b155d3d1aef54470df38b1d101dd01c000c0100100e64bcb74e201dd01000000000000f942")] + [InlineData(HistorianDataType.Double, 124.5, + "4f4e01003c0000003200" + "7f970bb0b5a7344bb7bf6899ff06d027" + "c07d5f23d701dd01" + "c000" + "c0100100" + "08eff1e6e701dd01" + "000000000000000000205f40")] + [InlineData(HistorianDataType.Int2, 1234d, + "4f4e0100360000002c00" + "35bb0ca5865a7f4498f06bf4e4d9ab23" + "f012f356d701dd01" + "c000" + "c0100100" + "9546841ae801dd01" + "00000000d204")] + [InlineData(HistorianDataType.Int4, 12345d, + "4f4e0100380000002e00" + "ca9735f7f841b244b56f9c14ccfeac32" + "b09bc72fd701dd01" + "c000" + "c0100100" + "104b59f3e701dd01" + "0000000039300000")] + [InlineData(HistorianDataType.UInt4, 305419896d, + "4f4e0100380000002e00" + "e7ae22d8e4cc65439ebd8bcb09402974" + "602d6663d701dd01" + "c000" + "c0100100" + "498af726e801dd01" + "0000000078563412")] + public void SerializeAddStreamValuesBuffer_MatchesCapturedNativeBuffer(HistorianDataType dataType, double value, string capturedHex) { - var tagGuid = new Guid("07311dd5-f8e9-4766-93b1-55d3d1aef544"); - DateTime sampleTime = DateTime.FromFileTimeUtc(0x01dd01d1b138df70); - DateTime receivedTime = DateTime.FromFileTimeUtc(0x01dd01e274cb4be6); + byte[] captured = Convert.FromHexString(capturedHex); + + // The GUID + both FILETIMEs sit at fixed offsets (the value width varies after them). + var tagGuid = new Guid(captured.AsSpan(10, 16).ToArray()); + long sampleFt = BinaryPrimitives.ReadInt64LittleEndian(captured.AsSpan(26, 8)); + long receivedFt = BinaryPrimitives.ReadInt64LittleEndian(captured.AsSpan(40, 8)); byte[] actual = HistorianHistoricalWriteProtocol.SerializeAddStreamValuesBuffer( - tagGuid, sampleTime, value: 124.5, receivedTime, quality: 192); + tagGuid, + dataType, + DateTime.FromFileTimeUtc(sampleFt), + value, + DateTime.FromFileTimeUtc(receivedFt), + quality: 192); - Assert.Equal(Convert.FromHexString(CapturedBufferHex), actual); + Assert.Equal(captured, actual); + } + + [Fact] + public void SerializeAddStreamValuesBuffer_UnsupportedType_Throws() + { + Assert.Throws(() => + HistorianHistoricalWriteProtocol.SerializeAddStreamValuesBuffer( + Guid.NewGuid(), HistorianDataType.SingleByteString, DateTime.UtcNow, 1.0, DateTime.UtcNow)); } [Fact] public void SerializeAddStreamValuesBuffer_HeaderDeclaresLengths() { byte[] buffer = HistorianHistoricalWriteProtocol.SerializeAddStreamValuesBuffer( - Guid.NewGuid(), DateTime.UtcNow, value: 1.0, DateTime.UtcNow); + Guid.NewGuid(), HistorianDataType.Float, DateTime.UtcNow, value: 1.0, DateTime.UtcNow); - Assert.Equal(56, buffer.Length); Assert.Equal(0x4E4F, BitConverter.ToUInt16(buffer, 0)); // "ON" Assert.Equal(1, BitConverter.ToUInt16(buffer, 2)); // sampleCount Assert.Equal((uint)buffer.Length, BitConverter.ToUInt32(buffer, 4)); diff --git a/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs b/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs index df43ac7..240cfe4 100644 --- a/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs +++ b/tools/AVEVA.Historian.Grpc2023CaptureHarness/Program.cs @@ -227,7 +227,17 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness string tagName = GetOption(args, "--tag") ?? "SdkM3CaptureSandbox"; bool create = args.Contains("--create"); bool commit = args.Contains("--commit"); - float sampleValue = float.TryParse(GetOption(args, "--value"), out float fv) ? fv : 123.0f; + string dataType = GetOption(args, "--data-type") ?? "Float"; // Float|Double|Int2|Int4|UInt4 + string rawValue = GetOption(args, "--value") ?? "123"; + // Box the value as the CLR type the HistorianDataValue expects for this tag type. + object sampleValue = dataType switch + { + "Double" => (object)double.Parse(rawValue), + "Int2" => (object)short.Parse(rawValue), + "Int4" => (object)int.Parse(rawValue), + "UInt4" => (object)uint.Parse(rawValue), + _ => (object)float.Parse(rawValue), + }; string? user = Environment.GetEnvironmentVariable("HISTORIAN_USER"); string? password = Environment.GetEnvironmentVariable("HISTORIAN_PASSWORD"); if (string.IsNullOrEmpty(user)) @@ -278,7 +288,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness object tag = Activator.CreateInstance(tagType)!; SetProp(tag, "TagName", tagName); TrySetProp(tag, "TagDescription", "histsdk M3 non-streamed-write capture sandbox"); - TrySetProp(tag, "TagDataType", Enum.Parse(tagDataTypeEnum, "Float", true)); + TrySetProp(tag, "TagDataType", Enum.Parse(tagDataTypeEnum, dataType, true)); TrySetProp(tag, "TagStorageType", Enum.Parse(tagStorageEnum, "Cyclic", true)); object addErr = Activator.CreateInstance(errorType)!; object?[] addArgs = { tag, 0u, addErr }; @@ -332,7 +342,7 @@ namespace AVEVA.Historian.Grpc2023CaptureHarness // --- build the historical (backfill) value --- object value = Activator.CreateInstance(valueType)!; SetProp(value, "TagKey", tagKey); - TrySetProp(value, "DataValueType", Enum.Parse(valueDataTypeEnum, "Float", true)); + TrySetProp(value, "DataValueType", Enum.Parse(valueDataTypeEnum, dataType, true)); TrySetProp(value, "OpcQuality", (ushort)192); TrySetProp(value, "Value", sampleValue); DateTime ts = DateTime.UtcNow.AddHours(-2); // backfill = past timestamp