using System.Globalization;
using System.Text.Json;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Opc.Ua;
using Opc.Ua.Client;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
using static ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests.ServerEventRig;
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests;
///
/// Gitea #801 (SDK 2 live gate, "real AVEVA history"): the booted OtOpcUa server, with the production
/// Gateway backend ( → GatewayHistorianDataSource) pointed at a REAL
/// HistorianGateway in front of the real AVEVA Historian, serves HistoryRead over a real SDK client session.
/// Every wire answer is compared with the same backend's direct answer
/// (value, status, source timestamp), and the whole run is dumped as JSON so the same queries can be diffed
/// against another SDK build and against direct SQL on the historian's Runtime DB.
///
///
/// Read-only by construction. The Gateway backend has no (its
/// provider returns none), no provisioner, recorder or alarm writer is built here, and the one HistoryUpdate this
/// test sends targets a node bound to the designated write-sandbox tag (HISTGW_WRITE_SANDBOX_TAG) and must be
/// refused by the server before any backend is reached. Without that variable the HistoryUpdate case is skipped.
/// Env (skips when the endpoint or key is unset):HISTGW_GATEWAY_ENDPOINT (e.g.
/// http://wonder-app-vd03.zmr.zimmer.com:5221), HISTGW_GATEWAY_APIKEY (a key with historian:read;
/// never logged), optional HISTGW_GATEWAY_ALLOW_UNTRUSTED, HISTGW_WRITE_SANDBOX_TAG,
/// OTOPCUA_AVEVA_GATE_OUT (output directory for aveva-gate-*.json), OTOPCUA_AVEVA_GATE_LABEL
/// (a run label recorded in the output, e.g. the SDK version), OTOPCUA_AVEVA_GATE_TAGS (comma list of
/// historized analog tags), OTOPCUA_AVEVA_GATE_WINDOWS (start/end;start/end, ISO-8601 UTC),
/// OTOPCUA_AVEVA_GATE_EVENT_SOURCES (comma list), OTOPCUA_AVEVA_GATE_EVENT_WINDOW and
/// OTOPCUA_AVEVA_GATE_GALAXY_REFS (#886: comma list of Galaxy alarm references whose native conditions are wired to
/// one equipment folder that is not named after any of their objects). The defaults are the DEV
/// (wonder-app-vd03 / wonder-sql-vd03) tags and fixed past windows used for the 2026-11-12 run.
/// The local deterministic fixtures are NOT this test; only a real gateway in front of a real AVEVA
/// Historian makes its output AVEVA evidence.
///
public sealed class AvevaGatewayHistoryLiveGateTests
{
private static readonly string[] DefaultTags = ["ZIPSEN.WTC1", "ZIPSEN.OvenPressureTransducer", "SysPerfCPUTotal"];
// W1 spans the 2026-11-10 11:32:36Z platform stop on DEV (a stored NULL / Bad row, then a gap and a
// recovery row); W2 is a quiet stretch. Both are in the past, so every run (and every SDK build) sees the same data.
private const string DefaultWindows = "2026-11-10T11:30:00Z/2026-11-10T11:41:00Z;2026-11-11T14:00:00Z/2026-11-11T14:10:00Z";
// AVEVA A&E source names as the gateway reports them. The first must have events in the window (asserted): the
// platform object, whose System.Stop / System.Start bracket the W1 outage. Since #882 its folder also serves the
// object's attribute alarms, which the gateway reports with SourceName "DevWinPlatform.CPULoadAvg" (Source_ProcessVariable;
// Source_Object is DevWinPlatform) — asserted against the unfiltered read. The second is that attribute name as a folder
// of its own: no attribute-level notifier exists, and the gateway (Source_Object = the object) has nothing under it, so
// it stays GoodNoData. Recorded (wire must equal the backend), not required to be non-empty.
private const string DefaultEventSources = "DevWinPlatform,DevWinPlatform.CPULoadAvg";
private const string DefaultEventWindow = "2026-11-05T15:00:00Z/2026-11-12T00:00:00Z";
// #886: a REAL equipment folder — its id is not a Galaxy object name — whose native conditions dial these Galaxy alarm
// references (attributeRef). Two objects with A&E history on DEV in the event window: the platform's CPU-load alarm
// and a ZIPSEN Left_001 machine alarm. The folder must serve both objects' events and attribute alarms.
private const string DefaultGalaxyAlarmRefs = "DevWinPlatform.CPULoadHiAlarm,Left_001.FRR_Warning";
private const string GalaxyEquipment = "EQ-886-ZipsenLeft";
private const string MissingTag = "OtOpcUa.Gate801.DoesNotExist";
private const double IntervalMs = 60_000;
// The ten per-sample aggregates the Gateway backend serves (#681) and a set it must refuse (time-weighted / never).
private static readonly (string Name, NodeId Id, HistoryAggregateType? Domain)[] Served =
[
("Average", ObjectIds.AggregateFunction_Average, HistoryAggregateType.Average),
("Count", ObjectIds.AggregateFunction_Count, HistoryAggregateType.Count),
("Minimum", ObjectIds.AggregateFunction_Minimum, HistoryAggregateType.Minimum),
("Maximum", ObjectIds.AggregateFunction_Maximum, HistoryAggregateType.Maximum),
("Range", ObjectIds.AggregateFunction_Range, HistoryAggregateType.Range),
("Start", ObjectIds.AggregateFunction_Start, HistoryAggregateType.Start),
("End", ObjectIds.AggregateFunction_End, HistoryAggregateType.End),
("Delta", ObjectIds.AggregateFunction_Delta, HistoryAggregateType.Delta),
("MinimumActualTime", ObjectIds.AggregateFunction_MinimumActualTime, HistoryAggregateType.MinimumActualTime),
("MaximumActualTime", ObjectIds.AggregateFunction_MaximumActualTime, HistoryAggregateType.MaximumActualTime),
];
private static readonly (string Name, NodeId Id)[] Refused =
[
("TimeAverage", ObjectIds.AggregateFunction_TimeAverage),
("Total", ObjectIds.AggregateFunction_Total),
("TimeAverage2", ObjectIds.AggregateFunction_TimeAverage2),
("Interpolative", ObjectIds.AggregateFunction_Interpolative),
("PercentGood", ObjectIds.AggregateFunction_PercentGood),
("DurationGood", ObjectIds.AggregateFunction_DurationGood),
("StandardDeviationSample", ObjectIds.AggregateFunction_StandardDeviationSample),
];
[Fact]
public async Task Real_AVEVA_history_served_over_the_wire_matches_the_gateway_backend()
{
var endpoint = Env("HISTGW_GATEWAY_ENDPOINT");
var apiKey = Env("HISTGW_GATEWAY_APIKEY");
if (endpoint is null || apiKey is null)
Assert.Skip("Set HISTGW_GATEWAY_ENDPOINT and HISTGW_GATEWAY_APIKEY to run the #801 real-AVEVA history gate.");
var ct = TestContext.Current.CancellationToken;
var label = Env("OTOPCUA_AVEVA_GATE_LABEL") ?? "unlabelled";
var tags = (Env("OTOPCUA_AVEVA_GATE_TAGS") ?? string.Join(',', DefaultTags))
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var windows = ParseWindows(Env("OTOPCUA_AVEVA_GATE_WINDOWS") ?? DefaultWindows);
var eventSources = (Env("OTOPCUA_AVEVA_GATE_EVENT_SOURCES") ?? DefaultEventSources)
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var eventWindow = ParseWindows(Env("OTOPCUA_AVEVA_GATE_EVENT_WINDOW") ?? DefaultEventWindow)[0];
var sandboxTag = Env("HISTGW_WRITE_SANDBOX_TAG");
var galaxyRefs = (Env("OTOPCUA_AVEVA_GATE_GALAXY_REFS") ?? DefaultGalaxyAlarmRefs)
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
// The production backend path: the same registry + provider + factory the Host uses for Backend=Gateway.
var options = new ServerHistorianOptions
{
Backend = "Gateway",
Enabled = true,
Endpoint = endpoint!,
ApiKey = apiKey!,
UseTls = endpoint!.StartsWith("https://", StringComparison.OrdinalIgnoreCase),
AllowUntrustedServerCertificate = string.Equals(Env("HISTGW_GATEWAY_ALLOW_UNTRUSTED"), "true", StringComparison.OrdinalIgnoreCase),
};
var provider = HistorianBackendRegistry.Resolve(options, new ConfigurationBuilder().Build());
provider.CreateUpdater(new ServiceCollection().BuildServiceProvider())
.ShouldBeNull("the Gateway backend must not offer HistoryUpdate — this gate never writes through it");
using var source = provider.CreateDataSource(new ServiceCollection().BuildServiceProvider());
var report = new Report(label, endpoint!);
var failures = new List();
await using var rig = await StartAsync("AvevaGate801", encrypted: false, ct);
rig.Nm.HistorianDataSource = source;
// The Host copies these from ServerHistorianOptions (OtOpcUaServerHostedService); defaults here.
rig.Nm.MaxTieClusterOverfetch = options.MaxTieClusterOverfetch;
rig.Nm.HonorReturnBounds = options.HonorReturnBounds;
rig.Nm.EventReadHorizon = options.EventReadHorizon;
rig.Nm.HistoryReadBatchParallelism = options.HistoryReadBatchParallelism;
rig.Nm.MaxConcurrentHistoryReadBatches = options.MaxConcurrentHistoryReadBatches;
rig.Nm.HistoryReadDeadline = options.HistoryReadDeadline;
rig.Enforce(false, [Key()]);
var nodes = new Dictionary(StringComparer.Ordinal);
for (var i = 0; i < tags.Length; i++) nodes[tags[i]] = rig.EnsureRawVariable($"Hist{i}", "Double", writable: false, tags[i]);
var missingNode = rig.EnsureRawVariable("HistMissing", "Double", writable: false, MissingTag);
var sysTimeNode = rig.EnsureRawVariable("HistSysTimeSec", "Double", writable: false, "SysTimeSec");
var sandboxNode = sandboxTag is null ? null : rig.EnsureRawVariable("HistSandbox", "Double", writable: false, sandboxTag);
// An event-history source: a folder whose id is the historian source name, promoted by a native condition.
foreach (var eventSource in eventSources)
{
rig.Sink.EnsureFolder(eventSource, parentNodeId: null, displayName: eventSource, realm: AddressSpaceRealm.Raw);
rig.Sink.MaterialiseAlarmCondition($"{eventSource}/gate801", eventSource, "gate801", "OffNormalAlarm", 500,
AddressSpaceRealm.Raw, isNative: true);
}
// #886: the applier's shape for a Galaxy deployment — each native condition under its raw device folder, bound to
// its Galaxy alarm reference and wired to the one equipment folder that references both.
rig.Sink.EnsureFolder(GalaxyEquipment, parentNodeId: null, displayName: "ZIPSEN Left line", realm: AddressSpaceRealm.Uns);
var galaxyConditions = new Dictionary(StringComparer.OrdinalIgnoreCase);
foreach (var reference in galaxyRefs)
{
var objectName = GalaxyAttributeRef.ObjectNameOf(reference)!;
var name = reference[(reference.IndexOf('.') + 1)..].Replace(".", "_", StringComparison.Ordinal);
var deviceFolder = $"Plant/Galaxy/{objectName}";
var rawPath = $"{deviceFolder}/{name}";
rig.Sink.EnsureFolder(deviceFolder, parentNodeId: null, displayName: objectName, realm: AddressSpaceRealm.Raw);
rig.Sink.MaterialiseAlarmCondition(rawPath, deviceFolder, name, "OffNormalAlarm", 500, AddressSpaceRealm.Raw, isNative: true);
rig.Sink.BindNativeAlarmHistory(rawPath, AddressSpaceRealm.Raw, reference);
rig.Sink.WireAlarmNotifiers(rawPath, AddressSpaceRealm.Raw, [GalaxyEquipment], AddressSpaceRealm.Uns);
galaxyConditions[reference] = (rawPath, name);
}
using var session = await rig.ConnectAsync(anonymous: true, ct);
var ns = (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri);
NodeId Node(string id) => new(id, ns);
// ---- Metadata: node attributes and the HistoryServerCapabilities the server advertises.
foreach (var (tag, id) in nodes.Take(1))
{
var attrs = await ReadAttributesAsync(session, Node(id), ct);
report.Add(new Case("metadata.node", tag, null, "wire", "Good", null, attrs));
Check(failures, attrs["Historizing"] == "Boolean:True", "Historizing must be true on a historized node");
var access = byte.Parse(attrs["AccessLevel"]!.Replace("Byte:", "", StringComparison.Ordinal), CultureInfo.InvariantCulture);
Check(failures, (access & AccessLevels.HistoryRead) != 0, "AccessLevel must carry HistoryRead");
Check(failures, (access & AccessLevels.HistoryWrite) == 0, "AccessLevel must not carry HistoryWrite on the Gateway backend");
}
var caps = await ReadHistoryCapabilitiesAsync(session, ct);
report.Add(new Case("metadata.historyServerCapabilities", null, null, "wire", "Good", null, caps));
Check(failures, caps["AggregateFunctions"] == string.Join(",", Served.Select(a => a.Name).OrderBy(n => n, StringComparer.Ordinal)),
$"advertised history aggregates must be exactly the Gateway's ten, got {caps["AggregateFunctions"]}");
// lmxopcua#881: the Boolean capabilities follow the Gateway backend — data + event reads, no HistoryUpdate path, no
// stored server timestamps, never event / annotation updates (this gate recorded all-false before the fix).
foreach (var flag in new[] { "AccessHistoryDataCapability", "AccessHistoryEventsCapability" })
Check(failures, caps.GetValueOrDefault(flag) == "Boolean:True", $"{flag} must be true on the Gateway backend, got {caps.GetValueOrDefault(flag)}");
foreach (var flag in new[]
{
"ServerTimestampSupported", "InsertDataCapability", "ReplaceDataCapability", "UpdateDataCapability",
"DeleteRawCapability", "DeleteAtTimeCapability", "InsertEventCapability", "ReplaceEventCapability",
"UpdateEventCapability", "DeleteEventCapability", "InsertAnnotationCapability",
})
Check(failures, caps.GetValueOrDefault(flag) == "Boolean:False", $"{flag} must be false on the Gateway backend, got {caps.GetValueOrDefault(flag)}");
foreach (var (tag, id) in nodes)
{
foreach (var (start, end) in windows)
{
var w = $"{start:o}/{end:o}";
// ---- Raw, all values, no bounds.
var wire = await ReadRawAsync(session, Node(id), start, end, 0, returnBounds: false, ct);
var direct = await source.ReadRawAsync(tag, start, end, 0, ct);
report.Add(new Case("raw.all", tag, w, "wire", wire.Status, null, null, wire.Rows));
report.Add(new Case("raw.all", tag, w, "direct", "Good", null, null, Rows(direct.Samples)));
CheckSame(failures, $"raw.all {tag} {w}", wire.Rows, Rows(direct.Samples));
Check(failures, wire.Rows.Count > 0, $"raw.all {tag} {w}: expected stored samples");
// ---- Raw, paged through continuation points (page 37).
var paged = await ReadRawPagedAsync(session, Node(id), start, end, 37, ct);
report.Add(new Case("raw.paged37", tag, w, "wire", paged.Status, $"pages={paged.Pages}", null, paged.Rows));
CheckSame(failures, $"raw.paged37 {tag} {w}", paged.Rows, wire.Rows);
// ---- Raw, reverse (StartTime = end > EndTime = start), first page of 25, newest first. Part 11: the
// StartTime is included and the EndTime excluded, so the reverse window is (start, end] — a sample stored
// exactly at `end` is in it (and not in the forward read). Reference: the backend's (start, end] forward.
var reverse = await ReadRawAsync(session, Node(id), end, start, 25, returnBounds: false, ct);
var reverseRef = await source.ReadRawAsync(tag, start.AddTicks(1), end.AddTicks(1), 0, ct);
report.Add(new Case("raw.reverse25", tag, w, "wire", reverse.Status, null, null, reverse.Rows));
CheckSame(failures, $"raw.reverse25 {tag} {w}", reverse.Rows,
Rows(reverseRef.Samples).AsEnumerable().Reverse().Take(25).ToList());
// ---- Raw with ReturnBounds (Part 11 bounding values; served by the server, #691).
var bounds = await ReadRawAsync(session, Node(id), start, end, 0, returnBounds: true, ct);
report.Add(new Case("raw.bounds", tag, w, "wire", bounds.Status, null, null, bounds.Rows));
Check(failures, bounds.Rows.Count >= wire.Rows.Count, $"raw.bounds {tag} {w}: fewer rows than raw");
// ---- Modified (unsupported on Gateway).
var modified = await ReadModifiedAsync(session, Node(id), start, end, ct);
report.Add(new Case("modified", tag, w, "wire", modified, null, null));
Check(failures, modified == "BadHistoryOperationUnsupported", $"modified {tag} {w}: got {modified}");
// ---- Processed: the ten served aggregates, wire vs the backend's own answer.
foreach (var (name, aggId, domain) in Served)
{
var p = await ReadProcessedAsync(session, Node(id), start, end, aggId, ct);
var d = await source.ReadProcessedAsync(tag, start, end, TimeSpan.FromMilliseconds(IntervalMs), domain!.Value, ct);
report.Add(new Case($"processed.{name}", tag, w, "wire", p.Status, null, null, p.Rows));
report.Add(new Case($"processed.{name}", tag, w, "direct", "Good", null, null, Rows(d.Samples)));
CheckSame(failures, $"processed.{name} {tag} {w}", p.Rows, Rows(d.Samples));
}
foreach (var (name, aggId) in Refused)
{
var p = await ReadProcessedAsync(session, Node(id), start, end, aggId, ct);
report.Add(new Case($"processed.refused.{name}", tag, w, "wire", p.Status, null, null, p.Rows));
Check(failures, p.Status == "BadAggregateNotSupported", $"processed.{name} {tag} {w}: got {p.Status}");
}
// ---- AtTime: one stored timestamp, a midpoint between two stored samples, and the window start.
var times = new List { start };
if (wire.Rows.Count >= 2)
{
times.Add(wire.Rows[1].Src!.Value);
times.Add(wire.Rows[0].Src!.Value + (wire.Rows[1].Src!.Value - wire.Rows[0].Src!.Value) / 2);
}
var at = await ReadAtTimeAsync(session, Node(id), times, ct);
var atDirect = await source.ReadAtTimeAsync(tag, times, TypeCode.Double, ct);
report.Add(new Case("attime", tag, w, "wire", at.Status, string.Join(",", times.Select(t => t.ToString("o"))), null, at.Rows));
report.Add(new Case("attime", tag, w, "direct", "Good", null, null, Rows(atDirect.Samples)));
CheckSame(failures, $"attime {tag} {w}", at.Rows, Rows(atDirect.Samples));
}
}
// ---- A tag the historian does not have (#653): no data, not an error.
{
var (start, end) = windows[0];
var r = await ReadRawAsync(session, Node(missingNode), start, end, 0, returnBounds: false, ct);
report.Add(new Case("raw.missingTag", MissingTag, $"{start:o}/{end:o}", "wire", r.Status, null, null, r.Rows));
Check(failures, r.Rows.Count == 0 && !r.Status.StartsWith("Bad", StringComparison.Ordinal), $"missing tag: {r.Status} rows={r.Rows.Count}");
}
// ---- SysTimeSec raw (historiangw#29 reported server-side timeouts on 2023 R2) — recorded, not asserted.
{
var (start, _) = windows[^1];
var sw = System.Diagnostics.Stopwatch.StartNew();
var r = await ReadRawAsync(session, Node(sysTimeNode), start, start.AddSeconds(10), 0, returnBounds: false, ct);
report.Add(new Case("raw.SysTimeSec.known29", "SysTimeSec", $"{start:o}/{start.AddSeconds(10):o}", "wire", r.Status,
$"elapsedMs={sw.ElapsedMilliseconds}", null, r.Rows));
}
// ---- Events: the historian's own A&E rows for one source, wire (whole + paged) vs the backend.
{
var (start, end) = eventWindow;
var w = $"{start:o}/{end:o}";
var discovered = await source.ReadEventsAsync(null, start, end, 0, ct);
report.Add(new Case("events.unfilteredBySource", null, w, "direct", "Good",
string.Join(",", discovered.Events.GroupBy(e => e.SourceName ?? "").OrderBy(g => g.Key, StringComparer.Ordinal)
.Select(g => $"{g.Key}={g.Count()}")), null));
foreach (var eventSource in eventSources)
{
var directEvents = await source.ReadEventsAsync(eventSource, start, end, 0, ct);
var directRows = directEvents.Events.Select(e => new EventRow(e.EventId, e.EventTimeUtc, e.ReceivedTimeUtc, e.SourceName,
e.Message, e.Severity)).ToList();
var all = await ReadEventsAsync(session, Node(eventSource), start, end, 0, ct);
var paged = await ReadEventsAsync(session, Node(eventSource), start, end, 1, ct);
var reverse = await ReadEventsAsync(session, Node(eventSource), end, start, 0, ct);
report.Add(new Case("events.all", eventSource, w, "wire", all.Status, $"pages={all.Pages}", null, null, all.Rows));
report.Add(new Case("events.paged1", eventSource, w, "wire", paged.Status, $"pages={paged.Pages}", null, null, paged.Rows));
report.Add(new Case("events.reverse", eventSource, w, "wire", reverse.Status, $"pages={reverse.Pages}", null, null, reverse.Rows));
report.Add(new Case("events.all", eventSource, w, "direct", "Good", null, null, null, directRows));
if (eventSource == eventSources[0])
{
Check(failures, directRows.Count > 0, $"events {eventSource} {w}: expected stored events");
// #882: the object's folder serves its own events AND its attribute alarms (