test: add isolated cross-version application compatibility harness
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,22 @@
|
||||
#Requires -Version 7.0
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs the production application SDK compatibility matrix in isolated processes.
|
||||
.DESCRIPTION
|
||||
Uses exact baseline host/client publish artifacts and builds the candidate from a
|
||||
separate checkout. Both SDK versions remain in separate processes. Evidence,
|
||||
artifact SHA-256 hashes, negotiated policies and individual results are retained.
|
||||
The output directory must not exist. No fleet endpoint or store is used.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)][string]$BaselineRoot,
|
||||
[Parameter(Mandatory)][string]$BaselineArtifacts,
|
||||
[Parameter(Mandatory)][string]$CandidateRoot,
|
||||
[Parameter(Mandatory)][string]$OutputDirectory,
|
||||
[string]$ContainerImage = 'mcr.microsoft.com/dotnet/sdk:10.0.103-noble'
|
||||
)
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$runner = Join-Path $PSScriptRoot '../validation/opcua-sdk-compatibility/run.py'
|
||||
& python3 $runner --baseline-root $BaselineRoot --baseline-artifacts $BaselineArtifacts --candidate-root $CandidateRoot --output $OutputDirectory --container-image $ContainerImage
|
||||
exit $LASTEXITCODE
|
||||
@@ -0,0 +1,374 @@
|
||||
#pragma warning disable OTOPCUA0001 // This wire harness deliberately exercises public driver capabilities directly.
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Opc.Ua;
|
||||
using Opc.Ua.Client;
|
||||
using ZB.MOM.WW.OtOpcUa.Client.Shared;
|
||||
using ZB.MOM.WW.OtOpcUa.Client.Shared.Models;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Authorization;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security;
|
||||
using AlarmEventArgs = ZB.MOM.WW.OtOpcUa.Client.Shared.Models.AlarmEventArgs;
|
||||
using WriteRequest = ZB.MOM.WW.OtOpcUa.Core.Abstractions.WriteRequest;
|
||||
using SharedSecurityMode = ZB.MOM.WW.OtOpcUa.Client.Shared.Models.SecurityMode;
|
||||
using CoreHistory = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult;
|
||||
|
||||
internal static class ProcessHost
|
||||
{
|
||||
static readonly DateTime Epoch = new(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
const string RawUri = "https://zb.com/otopcua/raw", UnsUri = "https://zb.com/otopcua/uns";
|
||||
const string Folder = "Matrix/dev", Scalar = Folder + "/Scalar", Vector = Folder + "/Vector", Alarm = Folder + "/Alarm";
|
||||
static readonly CancellationToken Ct = new CancellationTokenSource(TimeSpan.FromMinutes(4)).Token;
|
||||
static string Password => Environment.GetEnvironmentVariable("OPCUA_MATRIX_PASSWORD") ?? throw new InvalidOperationException("Missing ephemeral credential");
|
||||
static readonly List<object> Cases = [];
|
||||
static readonly List<object> Observations = [];
|
||||
static int Failures;
|
||||
static string Root = "";
|
||||
static object? Clr(DataValue v)
|
||||
{
|
||||
#if SDK2
|
||||
return ZB.MOM.WW.OtOpcUa.Client.Shared.Helpers.ValueConverter.ToClrObject(v.WrappedValue);
|
||||
#else
|
||||
return v.Value;
|
||||
#endif
|
||||
}
|
||||
static DateTime SourceTime(DataValue value)
|
||||
{
|
||||
#if SDK2
|
||||
return value.SourceTimestamp.ToDateTime();
|
||||
#else
|
||||
return value.SourceTimestamp;
|
||||
#endif
|
||||
}
|
||||
static DateTime ServerTime(DataValue value)
|
||||
{
|
||||
#if SDK2
|
||||
return value.ServerTimestamp.ToDateTime();
|
||||
#else
|
||||
return value.ServerTimestamp;
|
||||
#endif
|
||||
}
|
||||
static void Require(bool yes, string message) { if (!yes) throw new InvalidOperationException(message); }
|
||||
static async Task Case(string name, Func<Task> action)
|
||||
{
|
||||
try { await action(); Cases.Add(new { name, status = "passed" }); }
|
||||
catch (Exception e) { Failures++; Cases.Add(new { name, status = "failed", error = e.ToString() }); }
|
||||
}
|
||||
static async Task Wait(Func<bool> predicate)
|
||||
{
|
||||
for (int i = 0; i < 200; i++) { if (predicate()) return; await Task.Delay(50, Ct); }
|
||||
Require(false, "Notification deadline exceeded");
|
||||
}
|
||||
static ConnectionSettings Settings(string endpoint) => new()
|
||||
{
|
||||
EndpointUrl = endpoint,
|
||||
Username = "apikey",
|
||||
Password = Password,
|
||||
SecurityMode = SharedSecurityMode.SignAndEncrypt,
|
||||
AutoAcceptCertificates = false,
|
||||
CertificateStorePath = Environment.GetEnvironmentVariable("OPCUA_MATRIX_CLIENT_PKI") ?? Path.Combine(Root, "client-pki")
|
||||
};
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
Root = Path.GetFullPath(args[1]); Directory.CreateDirectory(Root);
|
||||
try
|
||||
{
|
||||
switch (args[0])
|
||||
{
|
||||
case "local-data-path": Console.WriteLine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)); return 0;
|
||||
case "prepare":
|
||||
var type = typeof(OpcUaClientService).Assembly.GetType("ZB.MOM.WW.OtOpcUa.Client.Shared.Adapters.DefaultApplicationConfigurationFactory")!;
|
||||
var factory = Activator.CreateInstance(type, true)!;
|
||||
await (Task<ApplicationConfiguration>)type.GetMethod("CreateAsync")!.Invoke(factory, [Settings("opc.tcp://127.0.0.1:1"), Ct])!;
|
||||
Console.WriteLine("PREPARED"); return 0;
|
||||
case "prepare-driver": await PrepareDriver(); return 0;
|
||||
case "driver": await Driver(args[2]); break;
|
||||
case "server": await Serve(); return 0;
|
||||
case "client": await Client(args[2]); break;
|
||||
case "performance": await Performance(args[2]); break;
|
||||
default: throw new ArgumentException("Expected prepare/server/client");
|
||||
}
|
||||
}
|
||||
catch (Exception e) { Failures++; Cases.Add(new { name = "process", status = "failed", error = e.ToString() }); }
|
||||
File.WriteAllText(Path.Combine(Root, "result.json"), JsonSerializer.Serialize(new { sdk = typeof(NodeId).Assembly.GetName().Version?.ToString(), process = Environment.ProcessId, cases = Cases, observations = Observations, failures = Failures, loadedSdkAssemblies = AppDomain.CurrentDomain.GetAssemblies().Where(a => a.GetName().Name!.StartsWith("Opc.Ua", StringComparison.Ordinal)).Select(a => new { name = a.GetName().Name, version = a.GetName().Version?.ToString(), path = a.Location }) }, new JsonSerializerOptions { WriteIndented = true }));
|
||||
return Failures == 0 ? 0 : 1;
|
||||
}
|
||||
static async Task Serve()
|
||||
{
|
||||
var options = new OpcUaApplicationHostOptions
|
||||
{
|
||||
ApplicationName = "OtOpcUa.Matrix",
|
||||
ApplicationUri = $"urn:OtOpcUa.Matrix.{Guid.NewGuid():N}",
|
||||
PublicHostname = "127.0.0.1",
|
||||
PkiStoreRoot = Path.Combine(Root, "server-pki"),
|
||||
AllowAnonymous = false,
|
||||
AutoAcceptUntrustedClientCertificates = false,
|
||||
EnabledSecurityProfiles = Environment.GetEnvironmentVariable("OPCUA_MATRIX_SERVER_SECURITY") == "none"
|
||||
? [OpcUaSecurityProfile.None] : [OpcUaSecurityProfile.Basic256Sha256SignAndEncrypt]
|
||||
};
|
||||
using (var listener = new TcpListener(IPAddress.Loopback, 0)) { listener.Start(); options.OpcUaPort = ((IPEndPoint)listener.LocalEndpoint).Port; }
|
||||
var server = new OtOpcUaSdkServer();
|
||||
await using var host = new OpcUaApplicationHost(options, NullLogger<OpcUaApplicationHost>.Instance, new Auth());
|
||||
await host.StartAsync(server, Ct);
|
||||
var nm = server.NodeManager!; var sink = new SdkAddressSpaceSink(nm);
|
||||
var authority = new DataPlaneAuthorityState();
|
||||
authority.Apply("matrix", "rev1", [new ApiKey { ApiKeyId = "matrix", Name = "matrix", SecretHash = "fixture", Principal = "matrix", Enabled = true, CreatedUtc = DateTime.UtcNow.AddDays(-1) }],
|
||||
[new NodeAcl { NodeAclId = "all", ClusterId = "matrix", Principal = "matrix", ScopeKind = NodeAclScopeKind.Cluster, PermissionFlags = (NodePermissions)int.MaxValue }], new Dictionary<AclScopeRef, IReadOnlyList<NodeAclPath>>());
|
||||
nm.DataPlaneAuthority = authority;
|
||||
nm.Authorizer = new DataPlaneAuthorizer(authority, new TriePermissionEvaluator(authority.Tries), true, NullLogger.Instance, TimeProvider.System);
|
||||
var history = new Source(Root); nm.HistorianDataSource = history;
|
||||
sink.EnsureFolder(Folder, null, "dev", AddressSpaceRealm.Raw);
|
||||
sink.EnsureFolder("Eq", null, "Eq", AddressSpaceRealm.Uns);
|
||||
foreach (var realm in new[] { AddressSpaceRealm.Raw, AddressSpaceRealm.Uns })
|
||||
{
|
||||
var scalar = realm == AddressSpaceRealm.Raw ? Scalar : "Eq/Scalar";
|
||||
var vector = realm == AddressSpaceRealm.Raw ? Vector : "Eq/Vector";
|
||||
var parent = realm == AddressSpaceRealm.Raw ? Folder : "Eq";
|
||||
sink.EnsureVariable(scalar, parent, "Scalar", "Int32", true, realm, "matrix.scalar");
|
||||
sink.EnsureVariable(vector, parent, "Vector", "Int32", true, realm, isArray: true, arrayLength: 3);
|
||||
sink.WriteValue(scalar, 17, 0, Epoch, realm); sink.WriteValue(vector, new[] { 1, 2, 3 }, 0, Epoch, realm);
|
||||
}
|
||||
sink.AddReference("Eq/Scalar", AddressSpaceRealm.Uns, Scalar, AddressSpaceRealm.Raw);
|
||||
sink.MaterialiseAlarmCondition(Alarm, Folder, "Alarm", "OffNormalAlarm", 800, AddressSpaceRealm.Raw, isNative: true);
|
||||
sink.WireAlarmNotifiers(Alarm, AddressSpaceRealm.Raw, ["Eq"], AddressSpaceRealm.Uns);
|
||||
sink.EnsureVariable(Folder + "/UnsetTimestamp", Folder, "UnsetTimestamp", "Int32", false, AddressSpaceRealm.Raw);
|
||||
sink.WriteValue(Folder + "/UnsetTimestamp", 3, 0, DateTime.MinValue, AddressSpaceRealm.Raw);
|
||||
nm.NodeWriteGateway = new Gateway();
|
||||
var state = new AlarmConditionSnapshot(true, false, true, true, AlarmShelvingKind.Unshelved, 800, "matrix-alarm");
|
||||
nm.NativeAlarmAckRouter = _ => Task.Run(() => { state = state with { Acknowledged = true }; sink.WriteAlarmCondition(Alarm, state, DateTime.UtcNow, AddressSpaceRealm.Raw); });
|
||||
File.WriteAllText(Path.Combine(Root, "endpoint.txt"), $"opc.tcp://127.0.0.1:{options.OpcUaPort}/OtOpcUa");
|
||||
File.WriteAllText(Path.Combine(Root, "ready.json"), JsonSerializer.Serialize(new { endpoint = $"opc.tcp://127.0.0.1:{options.OpcUaPort}/OtOpcUa", process = Environment.ProcessId, sdk = typeof(NodeId).Assembly.GetName().Version?.ToString(), rawUri = RawUri, unsUri = UnsUri }));
|
||||
int tick = 0;
|
||||
while (!File.Exists(Path.Combine(Root, "stop")))
|
||||
{
|
||||
sink.WriteAlarmCondition(Alarm, state with { Message = ++tick % 2 == 0 ? "matrix-alarm" : "matrix-tick" }, DateTime.UtcNow, AddressSpaceRealm.Raw);
|
||||
await Task.Delay(1000, Ct);
|
||||
}
|
||||
using var process = System.Diagnostics.Process.GetCurrentProcess();
|
||||
File.WriteAllText(Path.Combine(Root, "server-resources.json"), JsonSerializer.Serialize(new { process.WorkingSet64, process.PrivateMemorySize64, process.HandleCount, threads=process.Threads.Count, cpuMilliseconds=process.TotalProcessorTime.TotalMilliseconds, allocatedBytes=GC.GetTotalAllocatedBytes(), collections=new[]{GC.CollectionCount(0),GC.CollectionCount(1),GC.CollectionCount(2)} }, new JsonSerializerOptions {WriteIndented=true}));
|
||||
}
|
||||
static async Task Client(string endpoint)
|
||||
{
|
||||
using var client = new OpcUaClientService();
|
||||
await Case("trusted-SignAndEncrypt-API-key", async () => { var info = await client.ConnectAsync(Settings(endpoint), Ct); Require(info.SecurityMode == "SignAndEncrypt" && info.SecurityPolicyUri == SecurityPolicies.Basic256Sha256, "Unexpected negotiated policy"); Observations.Add(new { info.SecurityMode, info.SecurityPolicyUri }); });
|
||||
if (!client.IsConnected) return;
|
||||
var namespaces = (string[])Clr(await client.ReadValueAsync(VariableIds.Server_NamespaceArray, Ct))!;
|
||||
var raw = (ushort)Array.IndexOf(namespaces, RawUri); var uns = (ushort)Array.IndexOf(namespaces, UnsUri);
|
||||
NodeId ScalarId(ushort ns) => new(ns == raw ? Scalar : "Eq/Scalar", ns);
|
||||
NodeId VectorId(ushort ns) => new(ns == raw ? Vector : "Eq/Vector", ns);
|
||||
await Case("raw-UNS-URI-browse", async () =>
|
||||
{
|
||||
Require(raw > 0 && uns > 0, "Missing namespace URI");
|
||||
var rawChildren = await client.BrowseAsync(new NodeId(Folder, raw), Ct);
|
||||
var unsChildren = await client.BrowseAsync(new NodeId("Eq", uns), Ct);
|
||||
Observations.Add(new { rawChildren, unsChildren });
|
||||
// Supplement the production Shared results with unfiltered wire-reference metadata.
|
||||
var adapter = typeof(OpcUaClientService).GetField("_session", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(client)!;
|
||||
var sdkSession = (Opc.Ua.Client.Session)adapter.GetType().GetField("_session", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(adapter)!;
|
||||
var (_, _, rawReferences) = await sdkSession.BrowseAsync(null, null, new NodeId(Folder, raw), 0u,
|
||||
BrowseDirection.Forward, ReferenceTypeIds.References, true, 0u);
|
||||
Observations.Add(new { rawReferences = (rawReferences.ToArray() ?? []).Select(reference => new { nodeId = reference.NodeId.ToString(), browseName = reference.BrowseName.ToString(), nodeClass = reference.NodeClass.ToString(), typeDefinition = reference.TypeDefinition.ToString(), referenceType = reference.ReferenceTypeId.ToString(), reference.IsForward }).ToArray() });
|
||||
var expectedRaw = new[] { Scalar, Vector, Alarm, Folder + "/UnsetTimestamp" }.Select(id => new NodeId(id, raw).ToString());
|
||||
Require(expectedRaw.All(id => rawChildren.Any(child => child.NodeId == id)), "Missing raw children: " + string.Join(",", expectedRaw.Where(id => !rawChildren.Any(child => child.NodeId == id))));
|
||||
Require(new[] { "Eq/Scalar", "Eq/Vector" }.Select(id => new NodeId(id, uns).ToString()).All(id => unsChildren.Any(child => child.NodeId == id)), "Missing UNS scalar/array children");
|
||||
});
|
||||
await Case("initial-quality-explicit-source-and-server-timestamps", async () => { var initial = await client.ReadValueAsync(ScalarId(raw), Ct); Require(StatusCode.IsGood(initial.StatusCode), "Bad initial quality"); Require(SourceTime(initial) == Epoch && ServerTime(initial) > Epoch, "Lost explicit initial timestamps"); Observations.Add(new { sourceTimestamp = SourceTime(initial).ToString("O"), serverTimestamp = ServerTime(initial).ToString("O"), contract = "Source supplied as 2026-10-01T00:00:00Z; server timestamp observed without normalization or historian algorithm changes" }); });
|
||||
foreach (var ns in new[] { raw, uns })
|
||||
{
|
||||
await Case($"scalar-read-write-ns-{ns}", async () => { Require(Clr(await client.ReadValueAsync(ScalarId(ns), Ct)) is int, "Scalar lost Int32 type"); Require(StatusCode.IsGood(await client.WriteValueAsync(ScalarId(ns), 42, Ct)), "Scalar write failed"); Require((int)Clr(await client.ReadValueAsync(ScalarId(ns), Ct))! == 42, "Scalar round trip failed"); });
|
||||
await Case($"typed-array-read-write-ns-{ns}", async () => { Require(Clr(await client.ReadValueAsync(VectorId(ns), Ct)) is int[], "Array lost Int32[] type"); Require(StatusCode.IsGood(await client.WriteValueAsync(VectorId(ns), new[] { 7, 8, 9 }, Ct)), "Array write failed"); Require(((int[])Clr(await client.ReadValueAsync(VectorId(ns), Ct))!).SequenceEqual(new[] { 7, 8, 9 }), "Array round trip failed"); });
|
||||
}
|
||||
await Case("monitor-quality-timestamps", async () => { var seen = new ConcurrentQueue<DataValue>(); client.DataChanged += (_, e) => seen.Enqueue(e.Value); await client.SubscribeAsync(ScalarId(raw), 100, Ct); await client.WriteValueAsync(ScalarId(raw), 73, Ct); await Wait(() => seen.Any(v => Equals(Clr(v), 73))); var v = seen.Last(v => Equals(Clr(v), 73)); Require(StatusCode.IsGood(v.StatusCode) && SourceTime(v) > Epoch && ServerTime(v) > Epoch, "Bad monitor quality or missing timestamps"); Observations.Add(new { sourceTimestamp = SourceTime(v).ToString("O"), serverTimestamp = ServerTime(v).ToString("O"), status = v.StatusCode.ToString() }); await client.UnsubscribeAsync(ScalarId(raw), Ct); });
|
||||
await Case("event-filter-and-native-ack", async () => { var alarms = new ConcurrentQueue<AlarmEventArgs>(); client.AlarmEvent += (_, a) => alarms.Enqueue(a); await client.SubscribeAlarmsAsync(ObjectIds.Server, 100, Ct); await Wait(() => alarms.Any(a => a.Message == "matrix-alarm" && a.ActiveState && a.Severity == 800)); var alarm = alarms.Last(a => a.Message == "matrix-alarm"); Require(alarm.EventId?.Length > 0, "Missing event ID"); Require(StatusCode.IsGood(await client.AcknowledgeAlarmAsync(new NodeId(Alarm, raw).ToString(), alarm.EventId!, "matrix-ack", Ct)), "Ack refused"); await Wait(() => alarms.Any(a => a.Message == "matrix-alarm" && a.AckedState)); await client.UnsubscribeAlarmsAsync(Ct); });
|
||||
await Case("raw-history-CP-paging", async () => { var values = await client.HistoryReadRawAsync(ScalarId(raw), Epoch, Epoch.AddMinutes(1), 100, Ct); Require(values.Count == 25, "Expected 25 raw samples, got " + values.Count); Require(values.Select(v => Convert.ToInt32(Clr(v))).SequenceEqual(Enumerable.Range(0, 25)), "Raw page loss/duplicates/order"); });
|
||||
await Case("event-history-CP-paging", async () =>
|
||||
{
|
||||
var filter = new EventFilter();
|
||||
#if SDK2
|
||||
filter.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId")) { AttributeId = Attributes.Value };
|
||||
#else
|
||||
filter.SelectClauses.Add(new SimpleAttributeOperand {TypeDefinitionId=ObjectTypeIds.BaseEventType,BrowsePath=new QualifiedNameCollection {new QualifiedName("EventId")},AttributeId=Attributes.Value});
|
||||
#endif
|
||||
var events = await client.HistoryReadEventsAsync(new NodeId(Folder, raw), Epoch, Epoch.AddMinutes(1), filter, 100, Ct); Require(events.Count == 25, "Expected 25 event samples, got " + events.Count);
|
||||
#if SDK2
|
||||
var ids = events.Select(e => System.Text.Encoding.UTF8.GetString((byte[])ZB.MOM.WW.OtOpcUa.Client.Shared.Helpers.ValueConverter.ToClrObject(e.Fields[0])!));
|
||||
#else
|
||||
var ids=events.Select(e=>System.Text.Encoding.UTF8.GetString((byte[])e.Fields[0].Value));
|
||||
#endif
|
||||
Require(ids.SequenceEqual(Enumerable.Range(0, 25).Select(i => $"e{i:00}")), "Event history loss/duplicates/order across timestamp ties");
|
||||
});
|
||||
await Case("event-history-where-filter-positive-and-negative", async () =>
|
||||
{
|
||||
foreach (var type in new[] { ObjectTypeIds.BaseEventType, ObjectTypeIds.AuditEventType })
|
||||
{
|
||||
var filter = new EventFilter();
|
||||
#if SDK2
|
||||
filter.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId")) { AttributeId = Attributes.Value };
|
||||
filter.WhereClause.Push(FilterOperator.OfType, Variant.From(type));
|
||||
#else
|
||||
filter.SelectClauses.Add(new SimpleAttributeOperand(ObjectTypeIds.BaseEventType,new QualifiedName("EventId")){AttributeId=Attributes.Value});
|
||||
filter.WhereClause.Push(FilterOperator.OfType,new LiteralOperand(new Variant(type)));
|
||||
#endif
|
||||
var events = await client.HistoryReadEventsAsync(new NodeId(Folder, raw), Epoch, Epoch.AddMinutes(1), filter, 100, Ct);
|
||||
Require(events.Count == (type == ObjectTypeIds.BaseEventType ? 25 : 0), $"Where filter for {type} expected {(type == ObjectTypeIds.BaseEventType ? 25 : 0)} got {events.Count}");
|
||||
}
|
||||
});
|
||||
var unset = await client.ReadValueAsync(new NodeId(Folder + "/UnsetTimestamp", raw), Ct);
|
||||
Observations.Add(new { sourceTimestamp = SourceTime(unset).ToString("O"), serverTimestamp = ServerTime(unset).ToString("O"), timestampObservation = "Fixture source supplied DateTime.MinValue (unset). These are unmodified application/SDK wire observations; no historian algorithm or expected timestamp was changed." });
|
||||
await Benchmark(client, ScalarId(raw), VectorId(raw));
|
||||
await client.DisconnectAsync(Ct);
|
||||
}
|
||||
static OpcUaClientDriverOptions DriverOptions(string endpoint) => new()
|
||||
{
|
||||
EndpointUrl = endpoint,
|
||||
SecurityPolicy = OpcUaSecurityPolicy.Basic256Sha256,
|
||||
SecurityMode = OpcUaSecurityMode.SignAndEncrypt,
|
||||
AuthType = OpcUaAuthType.Username,
|
||||
Username = "apikey",
|
||||
Password = Password,
|
||||
AutoAcceptCertificates = false,
|
||||
RawTags = [new("scalar",JsonSerializer.Serialize(new {nodeId=$"nsu={RawUri};s={Scalar}"}),true),
|
||||
new("vector",JsonSerializer.Serialize(new {nodeId=$"nsu={RawUri};s={Vector}"}),true),
|
||||
new("uns",JsonSerializer.Serialize(new {nodeId=$"nsu={UnsUri};s=Eq/Scalar"}),true)],
|
||||
SessionName = "MatrixDriver",
|
||||
ApplicationUri = "urn:localhost:MatrixDriver"
|
||||
};
|
||||
static string IsolatedLocalData()
|
||||
{
|
||||
var local = Path.GetFullPath(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData));
|
||||
Require(local.StartsWith(Path.GetFullPath(Root) + Path.DirectorySeparatorChar, StringComparison.Ordinal), "Driver/CLI canonical PKI must be in task-owned root: " + local);
|
||||
return local;
|
||||
}
|
||||
static async Task PrepareDriver()
|
||||
{
|
||||
var local = IsolatedLocalData();
|
||||
await using var driver = new OpcUaClientDriver(DriverOptions("opc.tcp://127.0.0.1:1"), "matrix");
|
||||
// The production configuration method creates the application's own certificate without
|
||||
// attempting a session. Reflection is confined to preparation; all wire operations below
|
||||
// call the unchanged public driver API.
|
||||
var type = typeof(OpcUaClientDriver);
|
||||
var hook = Activator.CreateInstance(type.GetNestedType("CertValidationHook", BindingFlags.NonPublic)!, true)!;
|
||||
await (Task<ApplicationConfiguration>)type.GetMethod("BuildApplicationConfigurationAsync", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(driver, [hook, Ct])!;
|
||||
File.WriteAllText(Path.Combine(Root, "driver-pki.txt"), Path.Combine(local, "OtOpcUa", "pki"));
|
||||
}
|
||||
static async Task Driver(string endpoint)
|
||||
{
|
||||
IsolatedLocalData();
|
||||
await using var driver = new OpcUaClientDriver(DriverOptions(endpoint), "matrix");
|
||||
await Case("driver-trusted-encrypted-API-key-connect", () => driver.InitializeAsync("{}", Ct));
|
||||
await Case("driver-raw-UNS-URI-scalar-and-array-read", async () =>
|
||||
{
|
||||
var values = await driver.ReadAsync(["scalar", "vector", "uns"], Ct);
|
||||
Require(values.Count == 3 && values.All(x => x.StatusCode == 0), "Read failure");
|
||||
Require(values[0].Value is int && values[1].Value is int[] && values[2].Value is int, "Lost scalar/array type");
|
||||
});
|
||||
await Case("driver-scalar-array-write-roundtrip", async () =>
|
||||
{
|
||||
var writes = await driver.WriteAsync([new WriteRequest("scalar", 91), new WriteRequest("vector", new[] { 11, 12, 13 })], Ct);
|
||||
Require(writes.All(x => x.StatusCode == 0), "Write failed");
|
||||
var values = await driver.ReadAsync(["scalar", "vector"], Ct);
|
||||
Require(Equals(values[0].Value, 91) && ((int[])values[1].Value!).SequenceEqual(new[] { 11, 12, 13 }), "Write round trip mismatch");
|
||||
});
|
||||
await Case("driver-monitor-quality-timestamps", async () =>
|
||||
{
|
||||
var seen = new ConcurrentQueue<DataValueSnapshot>();
|
||||
driver.OnDataChange += (_, e) => seen.Enqueue(e.Snapshot);
|
||||
var subscription = await driver.SubscribeAsync(["scalar"], TimeSpan.FromMilliseconds(100), Ct);
|
||||
await driver.WriteAsync([new WriteRequest("scalar", 92)], Ct);
|
||||
await Wait(() => seen.Any(x => Equals(x.Value, 92)));
|
||||
var value = seen.Last(x => Equals(x.Value, 92));
|
||||
Require(value.StatusCode == 0 && value.SourceTimestampUtc.HasValue && value.ServerTimestampUtc > Epoch, "Missing monitor quality/timestamps");
|
||||
Observations.Add(new { value.SourceTimestampUtc, value.ServerTimestampUtc, value.StatusCode });
|
||||
await driver.UnsubscribeAsync(subscription, Ct);
|
||||
});
|
||||
Observations.Add(new { capability = "driver-history", status = "not-applicable", evidence = "Driver has no IHistoryProvider/HistoryRead capability; .claude/rules/drivers.md #655. Shared/CLI own history." });
|
||||
await driver.ShutdownAsync(Ct);
|
||||
}
|
||||
static async Task Performance(string endpoint)
|
||||
{
|
||||
using var client = new OpcUaClientService();
|
||||
var info = await client.ConnectAsync(Settings(endpoint), Ct);
|
||||
Require(info.SecurityMode == "SignAndEncrypt" && info.SecurityPolicyUri == SecurityPolicies.Basic256Sha256, "Unexpected benchmark endpoint policy");
|
||||
var namespaces = (string[])Clr(await client.ReadValueAsync(VariableIds.Server_NamespaceArray, Ct))!;
|
||||
var raw = (ushort)Array.IndexOf(namespaces, RawUri);
|
||||
Require(raw > 0, "Missing benchmark raw namespace");
|
||||
Observations.Add(new {info.SecurityMode, info.SecurityPolicyUri, tieredCompilation=Environment.GetEnvironmentVariable("DOTNET_TieredCompilation"), scope="Two scalar/Int32[3] variables; zero data monitors during measurement; five fixture variables and one native condition"});
|
||||
await Case("warmed-production-client-read-write-measurement", () => Benchmark(client, new NodeId(Scalar, raw), new NodeId(Vector, raw)));
|
||||
await client.DisconnectAsync(Ct);
|
||||
}
|
||||
static async Task Benchmark(OpcUaClientService client, NodeId scalar, NodeId vector)
|
||||
{
|
||||
// Local diagnostic subset; excludes reconnect, history and load.
|
||||
var rounds = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_ROUNDS") ?? "3");
|
||||
var warmups = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_WARMUPS") ?? "10");
|
||||
var measured = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_MEASURED") ?? "50");
|
||||
Require(rounds > 0 && warmups >= 0 && measured > 0, "Invalid performance sample counts");
|
||||
for (int round = 1; round <= rounds; round++)
|
||||
{
|
||||
for (int warmup = 0; warmup < warmups; warmup++)
|
||||
{
|
||||
await client.ReadValueAsync(scalar, Ct); await client.WriteValueAsync(scalar, warmup, Ct);
|
||||
await client.ReadValueAsync(vector, Ct); await client.WriteValueAsync(vector, new[] { warmup, warmup + 1, warmup + 2 }, Ct);
|
||||
}
|
||||
var samples = new Dictionary<string, List<double>>();
|
||||
var before = System.Diagnostics.Process.GetCurrentProcess(); var cpu = before.TotalProcessorTime;
|
||||
var allocated = GC.GetTotalAllocatedBytes(); var gc = new[] { GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2) };
|
||||
foreach (var operation in new[] { "scalar-read", "scalar-write", "array-read", "array-write" })
|
||||
{
|
||||
var times = new List<double>();
|
||||
for (int i = 0; i < measured; i++)
|
||||
{
|
||||
var start = System.Diagnostics.Stopwatch.GetTimestamp();
|
||||
if (operation == "scalar-read") await client.ReadValueAsync(scalar, Ct);
|
||||
else if (operation == "scalar-write") Require(StatusCode.IsGood(await client.WriteValueAsync(scalar, i, Ct)), "Benchmark scalar write");
|
||||
else if (operation == "array-read") await client.ReadValueAsync(vector, Ct);
|
||||
else Require(StatusCode.IsGood(await client.WriteValueAsync(vector, new[] { i, i + 1, i + 2 }, Ct)), "Benchmark array write");
|
||||
times.Add(System.Diagnostics.Stopwatch.GetElapsedTime(start).TotalMilliseconds);
|
||||
}
|
||||
samples[operation] = times;
|
||||
}
|
||||
before.Refresh();
|
||||
Observations.Add(new
|
||||
{
|
||||
benchmarkRound = round,
|
||||
warmupOperationsPerKind = 10,
|
||||
operationsPerKind = 50,
|
||||
latencyMilliseconds = samples.ToDictionary(x => x.Key, x => new { p50 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.50) - 1), p95 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.95) - 1), p99 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.99) - 1), throughputOperationsPerSecond = measured * 1000.0 / x.Value.Sum() }),
|
||||
clientCpuMilliseconds = (before.TotalProcessorTime - cpu).TotalMilliseconds,
|
||||
clientAllocatedBytes = GC.GetTotalAllocatedBytes() - allocated, clientAllocatedBytesPerOperation = (GC.GetTotalAllocatedBytes() - allocated) / (4.0 * measured),
|
||||
clientWorkingSetBytes = before.WorkingSet64,
|
||||
clientPrivateBytes = before.PrivateMemorySize64,
|
||||
clientThreads = before.Threads.Count,
|
||||
clientHandles = before.HandleCount,
|
||||
collections = new[] { GC.CollectionCount(0) - gc[0], GC.CollectionCount(1) - gc[1], GC.CollectionCount(2) - gc[2] }
|
||||
});
|
||||
}
|
||||
}
|
||||
sealed class Auth : IOpcUaUserAuthenticator
|
||||
{
|
||||
public Task<OpcUaUserAuthResult> AuthenticateUserNameAsync(string username, string password, CancellationToken ct) => Task.FromResult(username == "apikey" && password == Password ? OpcUaUserAuthResult.AllowKey("matrix", [OpcUaDataPlaneRoles.WriteOperate, OpcUaDataPlaneRoles.AlarmAck], "matrix") : OpcUaUserAuthResult.Deny("Invalid ephemeral fixture key"));
|
||||
}
|
||||
sealed class Gateway : IOpcUaNodeWriteGateway { public Task<NodeWriteOutcome> WriteAsync(string id, object? value, AddressSpaceRealm realm, CancellationToken ct) => Task.FromResult(new NodeWriteOutcome(true, null)); }
|
||||
sealed class Source(string root) : IHistorianDataSource
|
||||
{
|
||||
void Log(string operation, int rows) => File.AppendAllText(Path.Combine(root, "history-requests.jsonl"), JsonSerializer.Serialize(new { operation, rows }) + "\n");
|
||||
public int MaxRawValuesPerRead => 7;
|
||||
public Task<CoreHistory> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct)
|
||||
{ var all = Enumerable.Range(0, 25).Select(i => new DataValueSnapshot(i, 0, Epoch.AddSeconds(i), Epoch.AddSeconds(i))).Where(v => v.SourceTimestampUtc >= s && v.SourceTimestampUtc < e).ToList(); var rows = all.Take((int)(m == 0 ? 7 : m)).ToArray(); Log("raw", rows.Length); return Task.FromResult(new CoreHistory(rows, all.Count > rows.Length ? [1] : null)); }
|
||||
public Task<HistoricalEventsResult> ReadEventsAsync(string? source, DateTime s, DateTime e, int max, CancellationToken ct)
|
||||
{ var all = Enumerable.Range(0, 25).Select(i => new HistoricalEvent($"e{i:00}", "matrix", Epoch.AddSeconds(i / 2), Epoch, "matrix-history", 500)).Where(v => v.EventTimeUtc >= s && v.EventTimeUtc < e).ToList(); var rows = all.Take(Math.Min(7, max <= 0 ? 7 : max)).ToArray(); Log("events", rows.Length); return Task.FromResult(new HistoricalEventsResult(rows, all.Count > rows.Length ? [1] : null)); }
|
||||
public Task<CoreHistory> ReadProcessedAsync(string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) => NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct);
|
||||
public Task<CoreHistory> ReadAtTimeAsync(string r, IReadOnlyList<DateTime> t, CancellationToken ct) => NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct);
|
||||
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
# OPC UA SDK process compatibility
|
||||
|
||||
Run from a dedicated candidate checkout with .NET 10, Python 3, PowerShell 7,
|
||||
and Docker. Supply the unchanged original application's existing `host/` and
|
||||
`client/` publish directories. The output directory must not already exist.
|
||||
|
||||
```powershell
|
||||
./scripts/e2e/test-opcua-sdk-compatibility.ps1 `
|
||||
-BaselineRoot /absolute/original-checkout `
|
||||
-BaselineArtifacts /absolute/original-publishes `
|
||||
-CandidateRoot /absolute/dedicated-candidate-checkout `
|
||||
-OutputDirectory /absolute/fresh-evidence-directory
|
||||
```
|
||||
|
||||
The baseline console references the exact published production assemblies;
|
||||
it does not rebuild or edit the original application. The candidate console
|
||||
references the production host, Client.Shared and OPC UA driver projects, and
|
||||
the candidate CLI is separately published. Different SDK versions always run
|
||||
in different processes. Source revisions, SDK assemblies, SHA-256 hashes,
|
||||
commands, per-case results, actual namespace URIs, negotiated Shared security,
|
||||
loaded client SDK assemblies, timestamps and backend request logs are retained.
|
||||
|
||||
The required Shared legs are original→candidate, candidate→original and
|
||||
candidate→candidate. A fourth original→original leg provides a reference for
|
||||
both the application contract and the small performance sample. Every Shared
|
||||
leg checks the exact scalar, array and condition browse identities, both
|
||||
namespaces by URI, typed scalar/Int32[3] read/write, monitored Good quality and
|
||||
source/server timestamps, the production alarm event filter and native
|
||||
acknowledgment, and complete raw/event history across continuation points.
|
||||
Event-history results include timestamp ties, exact IDs/order and positive and
|
||||
negative `OfType` predicates. The historian is a deterministic fixture behind
|
||||
the production server's session-bound continuation store; these results do
|
||||
not certify any SQL Server, QuestDB or AVEVA backend.
|
||||
|
||||
Server, Shared client and driver require trusted Basic256Sha256
|
||||
SignAndEncrypt peers, with certificate autoaccept disabled. Public certificates
|
||||
are exchanged before sessions open. The test authenticator accepts one exact,
|
||||
random, API-key-shaped credential from the process environment; production
|
||||
ACL, write and condition-method checks use the resulting key identity. This
|
||||
fixture does not replace cryptographic API-key-authenticator coverage.
|
||||
|
||||
The driver and actual CLI run in a disposable `--network none` Linux container,
|
||||
alongside a separate baseline or candidate server process. PKI lives only in
|
||||
the mounted task directory; no host profile PKI or fleet endpoint is touched.
|
||||
The driver hardcodes `LocalApplicationData/OtOpcUa/pki`, so Linux's task-local
|
||||
`XDG_DATA_HOME` supplies isolation. The helper refuses a driver store outside
|
||||
its task root. CLI uses its own task-local canonical store. CLI currently
|
||||
forces `AutoAcceptCertificates=true` and exposes no strict-validation flag
|
||||
(`CommandBase.CreateConnectionSettings`); the exchanged peers and encrypted
|
||||
server remain in use, but CLI is not counted as strict client validation.
|
||||
Driver history is N/A because the application removed that capability (#655);
|
||||
Shared and CLI execute history reads instead.
|
||||
|
||||
Three measured rounds follow ten warmup operations per kind in each round:
|
||||
50 scalar reads, scalar writes, array reads and array writes per kind. The
|
||||
sample records nearest-rank p50/p95/p99, sequential throughput, client CPU,
|
||||
allocated bytes, GC collections, working set, private bytes, threads and
|
||||
handles. This is a small local diagnostic on a shared development host. It
|
||||
has no production fan-out, reconnect, queue, backpressure, backend or soak
|
||||
performance claim and does not establish Gate G4.
|
||||
|
||||
A separately published `server` helper can be used by other local consumer
|
||||
checks. Supply a fresh directory and `OPCUA_MATRIX_PASSWORD` in the process
|
||||
environment, then launch `dotnet ProcessHost.dll server <directory>`.
|
||||
`ready.json`/`endpoint.txt` identify the ephemeral loopback port; touching
|
||||
`<directory>/stop` shuts it down. Its default is strict SignAndEncrypt only.
|
||||
`OPCUA_MATRIX_SERVER_SECURITY=none` is an explicit fixture-only override for
|
||||
checking an existing unsecured consumer subset; the matrix runner clears that
|
||||
override. Anonymous is disabled in either mode. Secure clients must place
|
||||
their public certificates under `server-pki/trusted/certs` before server start
|
||||
and trust the resulting `server-pki/own/certs` certificate.
|
||||
|
||||
Explicitly supplied timestamps are asserted. Unset source timestamps are
|
||||
recorded separately as observations, with their original application/SDK wire
|
||||
values. The harness does not change historian algorithms or normalize the
|
||||
unset-timestamp sentinel to manufacture parity.
|
||||
|
||||
For the supplementary sustained warmup experiment, use the built consoles from
|
||||
one completed matrix run:
|
||||
|
||||
```sh
|
||||
python3 scripts/validation/opcua-sdk-compatibility/performance.py \
|
||||
--baseline-harness /absolute/run/build-baseline/bin/Release/net10.0/ProcessHost.dll \
|
||||
--candidate-harness /absolute/run/build-candidate/bin/Release/net10.0/ProcessHost.dll \
|
||||
--output /absolute/fresh-performance-directory
|
||||
```
|
||||
|
||||
This runs three fresh baseline/candidate process pairs serially, with 1,000
|
||||
warmup and 2,000 measured operations per kind in each process. Both server and
|
||||
client receive `DOTNET_TieredCompilation=0` for this experiment only; the
|
||||
application's shipped configuration is unchanged. The recorded budget is a
|
||||
15% increase in the median of the three p95 samples for any operation kind.
|
||||
Exit code 2 means a diagnostic budget regression, while process failures
|
||||
remain errors. Even an experiment within that budget does not certify the
|
||||
complete production performance matrix. The allocation-per-operation field
|
||||
is the average across all four operation kinds in a measured workload.
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs only inside the disposable, network-isolated task container.
|
||||
set -euo pipefail
|
||||
mkdir -p /task/driver/data /task/cli/data /task/server
|
||||
XDG_DATA_HOME=/task/driver/data dotnet /candidate/ProcessHost.dll prepare-driver /task/driver > /task/driver-prepare.log 2>&1
|
||||
OPCUA_MATRIX_CLIENT_PKI=/task/cli/data/OtOpcUaClient/pki XDG_DATA_HOME=/task/cli/data dotnet /candidate/ProcessHost.dll prepare /task/cli > /task/cli-prepare.log 2>&1
|
||||
mkdir -p /task/server/server-pki/trusted/certs
|
||||
cp /task/driver/data/OtOpcUa/pki/own/certs/* /task/server/server-pki/trusted/certs/
|
||||
cp /task/cli/data/OtOpcUaClient/pki/own/certs/* /task/server/server-pki/trusted/certs/
|
||||
dotnet /server/ProcessHost.dll server /task/server > /task/server.log 2>&1 &
|
||||
server_pid=$!
|
||||
cleanup() {
|
||||
touch /task/server/stop
|
||||
wait "$server_pid" || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
for attempt in $(seq 1 600); do
|
||||
test ! -f /task/server/ready.json || break
|
||||
kill -0 "$server_pid" || exit 1
|
||||
sleep 0.1
|
||||
done
|
||||
test -f /task/server/ready.json
|
||||
endpoint=$(cat /task/server/endpoint.txt)
|
||||
mkdir -p /task/driver/data/OtOpcUa/pki/trusted/certs /task/cli/data/OtOpcUaClient/pki/trusted/certs
|
||||
cp /task/server/server-pki/own/certs/* /task/driver/data/OtOpcUa/pki/trusted/certs/
|
||||
cp /task/server/server-pki/own/certs/* /task/cli/data/OtOpcUaClient/pki/trusted/certs/
|
||||
result=0
|
||||
XDG_DATA_HOME=/task/driver/data dotnet /candidate/ProcessHost.dll driver /task/driver "$endpoint" > /task/driver.log 2>&1 || result=1
|
||||
# CLI currently autoaccepts by product design; it has no strict-validation flag.
|
||||
# Its public certificates are nevertheless explicitly exchanged in this fixture.
|
||||
XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll read --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev/Scalar' > /task/cli-read.log 2>&1 || result=1
|
||||
XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll historyread --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev/Scalar' --start 2026-10-01T00:00:00Z --end 2026-10-01T00:01:00Z --max 100 > /task/cli-raw-history.log 2>&1 || result=1
|
||||
XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll historyread --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev' --start 2026-10-01T00:00:00Z --end 2026-10-01T00:01:00Z --max 100 --events --fields EventId,Message > /task/cli-event-history.log 2>&1 || result=1
|
||||
exit "$result"
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Supplementary warmed, matched native-process experiment; not full Gate G4."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--baseline-harness', type=Path, required=True)
|
||||
parser.add_argument('--candidate-harness', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--tiered-compilation', choices=['0', '1'], default='0')
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32),
|
||||
DOTNET_TieredCompilation=args.tiered_compilation, OPCUA_MATRIX_PERF_ROUNDS='1',
|
||||
OPCUA_MATRIX_PERF_WARMUPS='1000', OPCUA_MATRIX_PERF_MEASURED='2000')
|
||||
for key in ('OPCUA_MATRIX_SERVER_SECURITY', 'OPCUA_MATRIX_CLIENT_PKI'):
|
||||
env.pop(key, None)
|
||||
metadata = {'scope': __doc__, 'environment': {k: env[k] for k in ('DOTNET_TieredCompilation', 'OPCUA_MATRIX_PERF_ROUNDS', 'OPCUA_MATRIX_PERF_WARMUPS', 'OPCUA_MATRIX_PERF_MEASURED')},
|
||||
'dotnetSdk': subprocess.check_output(['dotnet', '--version'], text=True).strip(), 'p95BudgetPercent': 15,
|
||||
'dateUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'logicalCpuCount': os.cpu_count()}
|
||||
commands, results, hashes = [], [], []
|
||||
|
||||
def write(path, data):
|
||||
path.write_text(json.dumps(data, indent=2) + '\n')
|
||||
|
||||
def run(command, log):
|
||||
commands.append({'argv': list(map(str, command)), 'log': str(log)})
|
||||
with log.open('w') as stream:
|
||||
result = subprocess.run(list(map(str, command)), stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=240)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f'Process exited {result.returncode}; {log}')
|
||||
|
||||
def copy_certs(source, destination):
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
files = list(source.glob('*'))
|
||||
if not files:
|
||||
raise RuntimeError('Missing public certificates: ' + str(source))
|
||||
for path in files:
|
||||
shutil.copy2(path, destination / path.name)
|
||||
|
||||
try:
|
||||
for pair in range(1, 4):
|
||||
for version, harness in [('baseline', args.baseline_harness.resolve()), ('candidate', args.candidate_harness.resolve())]:
|
||||
leg = output / f'pair-{pair}-{version}'
|
||||
leg.mkdir()
|
||||
client, server = leg / 'client', leg / 'server'
|
||||
client.mkdir()
|
||||
server.mkdir()
|
||||
run(['dotnet', harness, 'prepare', client], leg / 'prepare.log')
|
||||
copy_certs(client / 'client-pki/own/certs', server / 'server-pki/trusted/certs')
|
||||
server_log = (leg / 'server.log').open('w')
|
||||
command = ['dotnet', str(harness), 'server', str(server)]
|
||||
commands.append({'argv': command, 'log': str(leg / 'server.log')})
|
||||
process = subprocess.Popen(command, env=env, stdout=server_log, stderr=subprocess.STDOUT)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while not (server / 'ready.json').exists():
|
||||
if process.poll() is not None or time.monotonic() > deadline:
|
||||
raise RuntimeError('Server failed to start: ' + str(leg / 'server.log'))
|
||||
time.sleep(.1)
|
||||
ready = json.loads((server / 'ready.json').read_text())
|
||||
copy_certs(server / 'server-pki/own/certs', client / 'client-pki/trusted/certs')
|
||||
run(['dotnet', harness, 'performance', client, ready['endpoint']], leg / 'client.log')
|
||||
result = json.loads((client / 'result.json').read_text())
|
||||
result.update({'pair': pair, 'version': version, 'server': ready})
|
||||
finally:
|
||||
(server / 'stop').touch()
|
||||
try:
|
||||
process.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
server_log.close()
|
||||
result['serverResources'] = json.loads((server / 'server-resources.json').read_text())
|
||||
results.append(result)
|
||||
write(leg / 'measurement.json', result)
|
||||
summary = []
|
||||
for operation in ('scalar-read', 'scalar-write', 'array-read', 'array-write'):
|
||||
samples = {}
|
||||
for version in ('baseline', 'candidate'):
|
||||
rounds = [o for r in results if r['version'] == version for o in r['observations'] if 'benchmarkRound' in o]
|
||||
samples[version] = {'p95Milliseconds': [r['latencyMilliseconds'][operation]['p95'] for r in rounds],
|
||||
'allocatedBytesPerOperation': [r['clientAllocatedBytesPerOperation'] for r in rounds],
|
||||
'clientWorkingSetBytes': [r['clientWorkingSetBytes'] for r in rounds],
|
||||
'serverWorkingSetBytes': [r['serverResources']['WorkingSet64'] for r in results if r['version'] == version]}
|
||||
ratio = statistics.median(samples['candidate']['p95Milliseconds']) / statistics.median(samples['baseline']['p95Milliseconds'])
|
||||
summary.append({'operation': operation, 'samples': samples, 'medianP95ChangePercent': 100 * (ratio - 1), 'status': 'diagnostic-regression' if ratio > 1.15 else 'diagnostic-within-budget'})
|
||||
metadata['summary'] = summary
|
||||
metadata['status'] = 'diagnostic-regression' if any(r['status'] == 'diagnostic-regression' for r in summary) else 'diagnostic-within-budget'
|
||||
for row in summary:
|
||||
print(f"{row['operation']}: median p95 {row['medianP95ChangePercent']:+.1f}% ({row['status']})")
|
||||
finally:
|
||||
for harness in (args.baseline_harness.resolve(), args.candidate_harness.resolve()):
|
||||
for path in [harness, *harness.parent.glob('Opc.Ua*.dll'), *harness.parent.glob('ZB.MOM.WW.OtOpcUa*.dll'), *harness.parent.glob('OtOpcUa.Host.dll')]:
|
||||
hashes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'bytes': path.stat().st_size})
|
||||
write(output / 'artifact-hashes.json', hashes)
|
||||
write(output / 'commands.json', commands)
|
||||
write(output / 'measurements.json', results)
|
||||
write(output / 'summary.json', metadata)
|
||||
return 2 if metadata.get('status') == 'diagnostic-regression' else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Separate-process production application compatibility gate (no mixed SDK process)."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
HOST = 'src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj'
|
||||
CLIENT = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.Shared/ZB.MOM.WW.OtOpcUa.Client.Shared.csproj'
|
||||
DRIVER = 'src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.csproj'
|
||||
CLI = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/ZB.MOM.WW.OtOpcUa.Client.CLI.csproj'
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
path.write_text(json.dumps(value, indent=2) + '\n')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--baseline-root', required=True, type=Path)
|
||||
parser.add_argument('--baseline-artifacts', required=True, type=Path, help='Existing immutable host/ and client/ publishes')
|
||||
parser.add_argument('--candidate-root', required=True, type=Path, help='Dedicated candidate checkout; this script builds it')
|
||||
parser.add_argument('--container-image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble', help='Disposable local driver/CLI isolation; must supply .NET 10 and bash')
|
||||
parser.add_argument('--output', required=True, type=Path, help='Fresh nonexistent evidence directory')
|
||||
args = parser.parse_args()
|
||||
args.baseline_root = args.baseline_root.resolve()
|
||||
args.candidate_root = args.candidate_root.resolve()
|
||||
args.baseline_artifacts = args.baseline_artifacts.resolve()
|
||||
args.output = args.output.resolve()
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
source_info = {}
|
||||
for name, root in [('baseline', args.baseline_root), ('candidate', args.candidate_root)]:
|
||||
source_info[name] = {'root': str(root), 'commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip(),
|
||||
'sdkPackages': str(root / 'Directory.Packages.props')}
|
||||
source_info['scope'] = 'Loopback application contract and 3 warmed 50-operation rounds per scalar/Int32[3] read/write kind; shared development host, no full load/reconnect/failover/backend/soak performance claim.'
|
||||
source_info['dateUtc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
write_json(args.output / 'source-info.json', source_info)
|
||||
commands = []
|
||||
matrix = []
|
||||
env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32))
|
||||
env.pop('OPCUA_MATRIX_CLIENT_PKI', None)
|
||||
env.pop('OPCUA_MATRIX_SERVER_SECURITY', None)
|
||||
|
||||
def run(command, log, process_env=env, timeout=300):
|
||||
commands.append({'argv': ['<ephemeral-key>' if x == env['OPCUA_MATRIX_PASSWORD'] else str(x) for x in command], 'log': str(log)})
|
||||
with log.open('w') as stream:
|
||||
result = subprocess.run([str(x) for x in command], stdout=stream, stderr=subprocess.STDOUT, env=process_env, timeout=timeout)
|
||||
return result.returncode
|
||||
|
||||
def build(version):
|
||||
folder = args.output / ('build-' + version)
|
||||
folder.mkdir()
|
||||
project = ET.Element('Project', Sdk='Microsoft.NET.Sdk')
|
||||
props = ET.SubElement(project, 'PropertyGroup')
|
||||
for k, v in {'OutputType': 'Exe', 'TargetFramework': 'net10.0', 'Nullable': 'enable', 'ImplicitUsings': 'enable', 'LangVersion': 'latest', 'DefineConstants': 'SDK2' if version == 'candidate' else ''}.items():
|
||||
ET.SubElement(props, k).text = v
|
||||
items = ET.SubElement(project, 'ItemGroup')
|
||||
ET.SubElement(items, 'FrameworkReference', Include='Microsoft.AspNetCore.App')
|
||||
if version == 'candidate':
|
||||
for path in (HOST, CLIENT, DRIVER):
|
||||
ET.SubElement(items, 'ProjectReference', Include=str(args.candidate_root / path))
|
||||
else:
|
||||
dlls = {}
|
||||
for path in ('host', 'client'):
|
||||
for dll in (args.baseline_artifacts / path).glob('*.dll'):
|
||||
dlls[dll.name] = dll
|
||||
for name, dll in sorted(dlls.items()):
|
||||
ref = ET.SubElement(items, 'Reference', Include=name[:-4])
|
||||
ET.SubElement(ref, 'HintPath').text = str(dll)
|
||||
ET.SubElement(ref, 'Private').text = 'true'
|
||||
ET.SubElement(items, 'Compile', Include=str(HERE / 'ProcessHost.cs'))
|
||||
csproj = folder / 'ProcessHost.csproj'
|
||||
ET.ElementTree(project).write(csproj, encoding='unicode')
|
||||
status = run(['dotnet', 'build', csproj, '-c', 'Release', '-m:1', '--nologo'], folder / 'build.log')
|
||||
if status:
|
||||
raise RuntimeError(f'{version} harness build failed ({status}); see {folder / "build.log"}')
|
||||
return folder / 'bin/Release/net10.0/ProcessHost.dll'
|
||||
|
||||
def certificates(source, target):
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
files = list(source.glob('*'))
|
||||
if not files:
|
||||
raise RuntimeError('No public certificates at ' + str(source))
|
||||
for file in files:
|
||||
shutil.copy2(file, target / file.name)
|
||||
|
||||
def receipt(paths):
|
||||
return [{'path': str(p), 'sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'bytes': p.stat().st_size} for p in sorted(set(paths)) if p.is_file()]
|
||||
|
||||
baseline = candidate = candidate_cli = None
|
||||
try:
|
||||
# Baseline builds only the portable harness against the exact published production DLLs.
|
||||
baseline = build('baseline')
|
||||
candidate = build('candidate')
|
||||
candidate_cli = args.output / 'candidate-cli'
|
||||
status = run(['dotnet', 'publish', args.candidate_root / CLI, '-c', 'Release', '-o', candidate_cli, '-m:1', '--nologo'], args.output / 'candidate-cli-build.log')
|
||||
if status:
|
||||
raise RuntimeError('Candidate CLI publish failed')
|
||||
for name, client, server in [('baseline-client-to-baseline-server', baseline, baseline), ('baseline-client-to-candidate-server', baseline, candidate), ('candidate-client-to-baseline-server', candidate, baseline), ('candidate-client-to-candidate-server', candidate, candidate)]:
|
||||
leg = args.output / name
|
||||
leg.mkdir()
|
||||
server_root = leg / 'server'
|
||||
client_root = leg / 'shared'
|
||||
server_root.mkdir()
|
||||
client_root.mkdir()
|
||||
status = run(['dotnet', client, 'prepare', client_root], leg / 'prepare.log')
|
||||
if status:
|
||||
matrix.append({'name': name, 'status': 'failed', 'stage': 'client-certificate-preparation'})
|
||||
continue
|
||||
certificates(client_root / 'client-pki/own/certs', server_root / 'server-pki/trusted/certs')
|
||||
server_log = (leg / 'server.log').open('w')
|
||||
command = ['dotnet', str(server), 'server', str(server_root)]
|
||||
commands.append({'argv': command, 'log': str(leg / 'server.log')})
|
||||
proc = subprocess.Popen(command, stdout=server_log, stderr=subprocess.STDOUT, env=env)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while not (server_root / 'ready.json').exists():
|
||||
if proc.poll() is not None or time.monotonic() > deadline:
|
||||
raise RuntimeError(f'{name}: server did not become ready; see server.log')
|
||||
time.sleep(.1)
|
||||
ready = json.loads((server_root / 'ready.json').read_text())
|
||||
certificates(server_root / 'server-pki/own/certs', client_root / 'client-pki/trusted/certs')
|
||||
status = run(['dotnet', client, 'client', client_root, ready['endpoint']], leg / 'shared.log')
|
||||
result = json.loads((client_root / 'result.json').read_text())
|
||||
history = [json.loads(x) for x in (server_root / 'history-requests.jsonl').read_text().splitlines()] if (server_root / 'history-requests.jsonl').exists() else []
|
||||
result.update({'name': name, 'server': ready, 'status': 'passed' if status == 0 else 'failed', 'historyBackendRequests': history, 'certificateValidation': 'explicit trusted peers; server and Shared client autoaccept disabled', 'authentication': 'ephemeral API-key-shaped credential, exact fixture authenticator; real authority/ACL/method gates'})
|
||||
write_json(leg / 'leg.json', result)
|
||||
matrix.append(result)
|
||||
except Exception as exc:
|
||||
matrix.append({'name': name, 'status': 'failed', 'error': str(exc)})
|
||||
finally:
|
||||
(server_root / 'stop').touch()
|
||||
try:
|
||||
proc.wait(timeout=15)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.terminate()
|
||||
proc.wait(timeout=10)
|
||||
server_log.close()
|
||||
image_info = subprocess.check_output(['docker', 'image', 'inspect', args.container_image, '--format', '{{json .}}'], text=True)
|
||||
image_data = json.loads(image_info)
|
||||
write_json(args.output / 'container-image.json', {k: image_data.get(k) for k in ('Id', 'RepoDigests', 'Architecture', 'Os')})
|
||||
for server_name, server in [('baseline', baseline), ('candidate', candidate)]:
|
||||
name = 'candidate-driver-cli-to-' + server_name + '-server'
|
||||
leg = args.output / name
|
||||
leg.mkdir()
|
||||
shutil.copy2(HERE / 'container-leg.sh', leg / 'run.sh')
|
||||
container_name = 'otopcua-sdk2-matrix-' + uuid.uuid4().hex[:12]
|
||||
command = ['docker', 'run', '--rm', '--name', container_name, '--network', 'none', '-e', 'OPCUA_MATRIX_PASSWORD',
|
||||
'-v', str(candidate.parent) + ':/candidate:ro', '-v', str(server.parent) + ':/server:ro',
|
||||
'-v', str(candidate_cli) + ':/cli:ro', '-v', str(leg) + ':/task', args.container_image, 'bash', '/task/run.sh']
|
||||
try:
|
||||
status = run(command, leg / 'container.log')
|
||||
finally:
|
||||
subprocess.run(['docker', 'rm', '-f', container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
if (leg / 'driver/result.json').exists():
|
||||
result = json.loads((leg / 'driver/result.json').read_text())
|
||||
else:
|
||||
result = {'cases': [], 'error': 'Driver result missing; inspect preparation/container/server logs'}
|
||||
for operation, expected in [('read', 'Value: 92'), ('raw-history', '25 values returned.'), ('event-history', '25 events returned.')]:
|
||||
log = leg / ('cli-' + operation + '.log')
|
||||
passed = log.exists() and expected in log.read_text()
|
||||
result['cases'].append({'name': 'production-CLI-' + operation, 'status': 'passed' if passed else 'failed', 'log': str(log), 'expectedOutput': expected})
|
||||
result.update({'name': name, 'status': 'passed' if status == 0 and all(x['status'] == 'passed' for x in result['cases']) else 'failed',
|
||||
'container': container_name, 'network': 'none; all peers in task container, separate SDK processes',
|
||||
'driverCertificateValidation': 'explicit trusted peers; autoaccept disabled', 'cliCertificateValidation': 'explicit exchanged peers; CLI always configures AutoAcceptCertificates=true (CommandBase.CreateConnectionSettings)',
|
||||
'server': json.loads((leg / 'server/ready.json').read_text()) if (leg / 'server/ready.json').exists() else None})
|
||||
write_json(leg / 'leg.json', result)
|
||||
matrix.append(result)
|
||||
|
||||
finally:
|
||||
paths = [p for p in (baseline, candidate, args.baseline_root / 'Directory.Packages.props', args.candidate_root / 'Directory.Packages.props', HERE / 'ProcessHost.cs', HERE / 'run.py', HERE / 'container-leg.sh') if p is not None]
|
||||
for folder in (args.baseline_artifacts / 'host', args.baseline_artifacts / 'client', candidate.parent if candidate else None, candidate_cli):
|
||||
if folder is not None:
|
||||
paths.extend(folder.glob('Opc.Ua*.dll'))
|
||||
paths.extend(folder.glob('*.deps.json'))
|
||||
paths.extend(folder.glob('ZB.MOM.WW.OtOpcUa*.dll'))
|
||||
paths.extend(folder.glob('OtOpcUa.Host.dll'))
|
||||
paths.extend(folder.glob('*.runtimeconfig.json'))
|
||||
write_json(args.output / 'artifact-hashes.json', receipt(paths))
|
||||
write_json(args.output / 'commands.json', commands)
|
||||
write_json(args.output / 'matrix.json', matrix)
|
||||
return 1 if not matrix or any(x['status'] != 'passed' for x in matrix) else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user