diff --git a/scripts/e2e/test-opcua-sdk-compatibility.ps1 b/scripts/e2e/test-opcua-sdk-compatibility.ps1 new file mode 100644 index 00000000..49f92a60 --- /dev/null +++ b/scripts/e2e/test-opcua-sdk-compatibility.ps1 @@ -0,0 +1,22 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS +Runs the production application SDK compatibility matrix in isolated processes. +.DESCRIPTION +Uses exact baseline host/client publish artifacts and builds the candidate from a +separate checkout. Both SDK versions remain in separate processes. Evidence, +artifact SHA-256 hashes, negotiated policies and individual results are retained. +The output directory must not exist. No fleet endpoint or store is used. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)][string]$BaselineRoot, + [Parameter(Mandatory)][string]$BaselineArtifacts, + [Parameter(Mandatory)][string]$CandidateRoot, + [Parameter(Mandatory)][string]$OutputDirectory, + [string]$ContainerImage = 'mcr.microsoft.com/dotnet/sdk:10.0.103-noble' +) +$ErrorActionPreference = 'Stop' +$runner = Join-Path $PSScriptRoot '../validation/opcua-sdk-compatibility/run.py' +& python3 $runner --baseline-root $BaselineRoot --baseline-artifacts $BaselineArtifacts --candidate-root $CandidateRoot --output $OutputDirectory --container-image $ContainerImage +exit $LASTEXITCODE diff --git a/scripts/validation/opcua-sdk-compatibility/ProcessHost.cs b/scripts/validation/opcua-sdk-compatibility/ProcessHost.cs new file mode 100644 index 00000000..22a097f5 --- /dev/null +++ b/scripts/validation/opcua-sdk-compatibility/ProcessHost.cs @@ -0,0 +1,374 @@ +#pragma warning disable OTOPCUA0001 // This wire harness deliberately exercises public driver capabilities directly. +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Reflection; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Opc.Ua; +using Opc.Ua.Client; +using ZB.MOM.WW.OtOpcUa.Client.Shared; +using ZB.MOM.WW.OtOpcUa.Client.Shared.Models; +using ZB.MOM.WW.OtOpcUa.Commons.OpcUa; +using ZB.MOM.WW.OtOpcUa.Configuration.Entities; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Core.Authorization; +using ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient; +using ZB.MOM.WW.OtOpcUa.OpcUaServer; +using ZB.MOM.WW.OtOpcUa.OpcUaServer.Security; +using AlarmEventArgs = ZB.MOM.WW.OtOpcUa.Client.Shared.Models.AlarmEventArgs; +using WriteRequest = ZB.MOM.WW.OtOpcUa.Core.Abstractions.WriteRequest; +using SharedSecurityMode = ZB.MOM.WW.OtOpcUa.Client.Shared.Models.SecurityMode; +using CoreHistory = ZB.MOM.WW.OtOpcUa.Core.Abstractions.HistoryReadResult; + +internal static class ProcessHost +{ + static readonly DateTime Epoch = new(2026, 10, 1, 0, 0, 0, DateTimeKind.Utc); + const string RawUri = "https://zb.com/otopcua/raw", UnsUri = "https://zb.com/otopcua/uns"; + const string Folder = "Matrix/dev", Scalar = Folder + "/Scalar", Vector = Folder + "/Vector", Alarm = Folder + "/Alarm"; + static readonly CancellationToken Ct = new CancellationTokenSource(TimeSpan.FromMinutes(4)).Token; + static string Password => Environment.GetEnvironmentVariable("OPCUA_MATRIX_PASSWORD") ?? throw new InvalidOperationException("Missing ephemeral credential"); + static readonly List Cases = []; + static readonly List Observations = []; + static int Failures; + static string Root = ""; + static object? Clr(DataValue v) + { +#if SDK2 + return ZB.MOM.WW.OtOpcUa.Client.Shared.Helpers.ValueConverter.ToClrObject(v.WrappedValue); +#else + return v.Value; +#endif + } + static DateTime SourceTime(DataValue value) + { +#if SDK2 + return value.SourceTimestamp.ToDateTime(); +#else + return value.SourceTimestamp; +#endif + } + static DateTime ServerTime(DataValue value) + { +#if SDK2 + return value.ServerTimestamp.ToDateTime(); +#else + return value.ServerTimestamp; +#endif + } + static void Require(bool yes, string message) { if (!yes) throw new InvalidOperationException(message); } + static async Task Case(string name, Func action) + { + try { await action(); Cases.Add(new { name, status = "passed" }); } + catch (Exception e) { Failures++; Cases.Add(new { name, status = "failed", error = e.ToString() }); } + } + static async Task Wait(Func predicate) + { + for (int i = 0; i < 200; i++) { if (predicate()) return; await Task.Delay(50, Ct); } + Require(false, "Notification deadline exceeded"); + } + static ConnectionSettings Settings(string endpoint) => new() + { + EndpointUrl = endpoint, + Username = "apikey", + Password = Password, + SecurityMode = SharedSecurityMode.SignAndEncrypt, + AutoAcceptCertificates = false, + CertificateStorePath = Environment.GetEnvironmentVariable("OPCUA_MATRIX_CLIENT_PKI") ?? Path.Combine(Root, "client-pki") + }; + public static async Task Main(string[] args) + { + Root = Path.GetFullPath(args[1]); Directory.CreateDirectory(Root); + try + { + switch (args[0]) + { + case "local-data-path": Console.WriteLine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)); return 0; + case "prepare": + var type = typeof(OpcUaClientService).Assembly.GetType("ZB.MOM.WW.OtOpcUa.Client.Shared.Adapters.DefaultApplicationConfigurationFactory")!; + var factory = Activator.CreateInstance(type, true)!; + await (Task)type.GetMethod("CreateAsync")!.Invoke(factory, [Settings("opc.tcp://127.0.0.1:1"), Ct])!; + Console.WriteLine("PREPARED"); return 0; + case "prepare-driver": await PrepareDriver(); return 0; + case "driver": await Driver(args[2]); break; + case "server": await Serve(); return 0; + case "client": await Client(args[2]); break; + case "performance": await Performance(args[2]); break; + default: throw new ArgumentException("Expected prepare/server/client"); + } + } + catch (Exception e) { Failures++; Cases.Add(new { name = "process", status = "failed", error = e.ToString() }); } + File.WriteAllText(Path.Combine(Root, "result.json"), JsonSerializer.Serialize(new { sdk = typeof(NodeId).Assembly.GetName().Version?.ToString(), process = Environment.ProcessId, cases = Cases, observations = Observations, failures = Failures, loadedSdkAssemblies = AppDomain.CurrentDomain.GetAssemblies().Where(a => a.GetName().Name!.StartsWith("Opc.Ua", StringComparison.Ordinal)).Select(a => new { name = a.GetName().Name, version = a.GetName().Version?.ToString(), path = a.Location }) }, new JsonSerializerOptions { WriteIndented = true })); + return Failures == 0 ? 0 : 1; + } + static async Task Serve() + { + var options = new OpcUaApplicationHostOptions + { + ApplicationName = "OtOpcUa.Matrix", + ApplicationUri = $"urn:OtOpcUa.Matrix.{Guid.NewGuid():N}", + PublicHostname = "127.0.0.1", + PkiStoreRoot = Path.Combine(Root, "server-pki"), + AllowAnonymous = false, + AutoAcceptUntrustedClientCertificates = false, + EnabledSecurityProfiles = Environment.GetEnvironmentVariable("OPCUA_MATRIX_SERVER_SECURITY") == "none" + ? [OpcUaSecurityProfile.None] : [OpcUaSecurityProfile.Basic256Sha256SignAndEncrypt] + }; + using (var listener = new TcpListener(IPAddress.Loopback, 0)) { listener.Start(); options.OpcUaPort = ((IPEndPoint)listener.LocalEndpoint).Port; } + var server = new OtOpcUaSdkServer(); + await using var host = new OpcUaApplicationHost(options, NullLogger.Instance, new Auth()); + await host.StartAsync(server, Ct); + var nm = server.NodeManager!; var sink = new SdkAddressSpaceSink(nm); + var authority = new DataPlaneAuthorityState(); + authority.Apply("matrix", "rev1", [new ApiKey { ApiKeyId = "matrix", Name = "matrix", SecretHash = "fixture", Principal = "matrix", Enabled = true, CreatedUtc = DateTime.UtcNow.AddDays(-1) }], + [new NodeAcl { NodeAclId = "all", ClusterId = "matrix", Principal = "matrix", ScopeKind = NodeAclScopeKind.Cluster, PermissionFlags = (NodePermissions)int.MaxValue }], new Dictionary>()); + nm.DataPlaneAuthority = authority; + nm.Authorizer = new DataPlaneAuthorizer(authority, new TriePermissionEvaluator(authority.Tries), true, NullLogger.Instance, TimeProvider.System); + var history = new Source(Root); nm.HistorianDataSource = history; + sink.EnsureFolder(Folder, null, "dev", AddressSpaceRealm.Raw); + sink.EnsureFolder("Eq", null, "Eq", AddressSpaceRealm.Uns); + foreach (var realm in new[] { AddressSpaceRealm.Raw, AddressSpaceRealm.Uns }) + { + var scalar = realm == AddressSpaceRealm.Raw ? Scalar : "Eq/Scalar"; + var vector = realm == AddressSpaceRealm.Raw ? Vector : "Eq/Vector"; + var parent = realm == AddressSpaceRealm.Raw ? Folder : "Eq"; + sink.EnsureVariable(scalar, parent, "Scalar", "Int32", true, realm, "matrix.scalar"); + sink.EnsureVariable(vector, parent, "Vector", "Int32", true, realm, isArray: true, arrayLength: 3); + sink.WriteValue(scalar, 17, 0, Epoch, realm); sink.WriteValue(vector, new[] { 1, 2, 3 }, 0, Epoch, realm); + } + sink.AddReference("Eq/Scalar", AddressSpaceRealm.Uns, Scalar, AddressSpaceRealm.Raw); + sink.MaterialiseAlarmCondition(Alarm, Folder, "Alarm", "OffNormalAlarm", 800, AddressSpaceRealm.Raw, isNative: true); + sink.WireAlarmNotifiers(Alarm, AddressSpaceRealm.Raw, ["Eq"], AddressSpaceRealm.Uns); + sink.EnsureVariable(Folder + "/UnsetTimestamp", Folder, "UnsetTimestamp", "Int32", false, AddressSpaceRealm.Raw); + sink.WriteValue(Folder + "/UnsetTimestamp", 3, 0, DateTime.MinValue, AddressSpaceRealm.Raw); + nm.NodeWriteGateway = new Gateway(); + var state = new AlarmConditionSnapshot(true, false, true, true, AlarmShelvingKind.Unshelved, 800, "matrix-alarm"); + nm.NativeAlarmAckRouter = _ => Task.Run(() => { state = state with { Acknowledged = true }; sink.WriteAlarmCondition(Alarm, state, DateTime.UtcNow, AddressSpaceRealm.Raw); }); + File.WriteAllText(Path.Combine(Root, "endpoint.txt"), $"opc.tcp://127.0.0.1:{options.OpcUaPort}/OtOpcUa"); + File.WriteAllText(Path.Combine(Root, "ready.json"), JsonSerializer.Serialize(new { endpoint = $"opc.tcp://127.0.0.1:{options.OpcUaPort}/OtOpcUa", process = Environment.ProcessId, sdk = typeof(NodeId).Assembly.GetName().Version?.ToString(), rawUri = RawUri, unsUri = UnsUri })); + int tick = 0; + while (!File.Exists(Path.Combine(Root, "stop"))) + { + sink.WriteAlarmCondition(Alarm, state with { Message = ++tick % 2 == 0 ? "matrix-alarm" : "matrix-tick" }, DateTime.UtcNow, AddressSpaceRealm.Raw); + await Task.Delay(1000, Ct); + } + using var process = System.Diagnostics.Process.GetCurrentProcess(); + File.WriteAllText(Path.Combine(Root, "server-resources.json"), JsonSerializer.Serialize(new { process.WorkingSet64, process.PrivateMemorySize64, process.HandleCount, threads=process.Threads.Count, cpuMilliseconds=process.TotalProcessorTime.TotalMilliseconds, allocatedBytes=GC.GetTotalAllocatedBytes(), collections=new[]{GC.CollectionCount(0),GC.CollectionCount(1),GC.CollectionCount(2)} }, new JsonSerializerOptions {WriteIndented=true})); + } + static async Task Client(string endpoint) + { + using var client = new OpcUaClientService(); + await Case("trusted-SignAndEncrypt-API-key", async () => { var info = await client.ConnectAsync(Settings(endpoint), Ct); Require(info.SecurityMode == "SignAndEncrypt" && info.SecurityPolicyUri == SecurityPolicies.Basic256Sha256, "Unexpected negotiated policy"); Observations.Add(new { info.SecurityMode, info.SecurityPolicyUri }); }); + if (!client.IsConnected) return; + var namespaces = (string[])Clr(await client.ReadValueAsync(VariableIds.Server_NamespaceArray, Ct))!; + var raw = (ushort)Array.IndexOf(namespaces, RawUri); var uns = (ushort)Array.IndexOf(namespaces, UnsUri); + NodeId ScalarId(ushort ns) => new(ns == raw ? Scalar : "Eq/Scalar", ns); + NodeId VectorId(ushort ns) => new(ns == raw ? Vector : "Eq/Vector", ns); + await Case("raw-UNS-URI-browse", async () => + { + Require(raw > 0 && uns > 0, "Missing namespace URI"); + var rawChildren = await client.BrowseAsync(new NodeId(Folder, raw), Ct); + var unsChildren = await client.BrowseAsync(new NodeId("Eq", uns), Ct); + Observations.Add(new { rawChildren, unsChildren }); + // Supplement the production Shared results with unfiltered wire-reference metadata. + var adapter = typeof(OpcUaClientService).GetField("_session", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(client)!; + var sdkSession = (Opc.Ua.Client.Session)adapter.GetType().GetField("_session", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(adapter)!; + var (_, _, rawReferences) = await sdkSession.BrowseAsync(null, null, new NodeId(Folder, raw), 0u, + BrowseDirection.Forward, ReferenceTypeIds.References, true, 0u); + Observations.Add(new { rawReferences = (rawReferences.ToArray() ?? []).Select(reference => new { nodeId = reference.NodeId.ToString(), browseName = reference.BrowseName.ToString(), nodeClass = reference.NodeClass.ToString(), typeDefinition = reference.TypeDefinition.ToString(), referenceType = reference.ReferenceTypeId.ToString(), reference.IsForward }).ToArray() }); + var expectedRaw = new[] { Scalar, Vector, Alarm, Folder + "/UnsetTimestamp" }.Select(id => new NodeId(id, raw).ToString()); + Require(expectedRaw.All(id => rawChildren.Any(child => child.NodeId == id)), "Missing raw children: " + string.Join(",", expectedRaw.Where(id => !rawChildren.Any(child => child.NodeId == id)))); + Require(new[] { "Eq/Scalar", "Eq/Vector" }.Select(id => new NodeId(id, uns).ToString()).All(id => unsChildren.Any(child => child.NodeId == id)), "Missing UNS scalar/array children"); + }); + await Case("initial-quality-explicit-source-and-server-timestamps", async () => { var initial = await client.ReadValueAsync(ScalarId(raw), Ct); Require(StatusCode.IsGood(initial.StatusCode), "Bad initial quality"); Require(SourceTime(initial) == Epoch && ServerTime(initial) > Epoch, "Lost explicit initial timestamps"); Observations.Add(new { sourceTimestamp = SourceTime(initial).ToString("O"), serverTimestamp = ServerTime(initial).ToString("O"), contract = "Source supplied as 2026-10-01T00:00:00Z; server timestamp observed without normalization or historian algorithm changes" }); }); + foreach (var ns in new[] { raw, uns }) + { + await Case($"scalar-read-write-ns-{ns}", async () => { Require(Clr(await client.ReadValueAsync(ScalarId(ns), Ct)) is int, "Scalar lost Int32 type"); Require(StatusCode.IsGood(await client.WriteValueAsync(ScalarId(ns), 42, Ct)), "Scalar write failed"); Require((int)Clr(await client.ReadValueAsync(ScalarId(ns), Ct))! == 42, "Scalar round trip failed"); }); + await Case($"typed-array-read-write-ns-{ns}", async () => { Require(Clr(await client.ReadValueAsync(VectorId(ns), Ct)) is int[], "Array lost Int32[] type"); Require(StatusCode.IsGood(await client.WriteValueAsync(VectorId(ns), new[] { 7, 8, 9 }, Ct)), "Array write failed"); Require(((int[])Clr(await client.ReadValueAsync(VectorId(ns), Ct))!).SequenceEqual(new[] { 7, 8, 9 }), "Array round trip failed"); }); + } + await Case("monitor-quality-timestamps", async () => { var seen = new ConcurrentQueue(); client.DataChanged += (_, e) => seen.Enqueue(e.Value); await client.SubscribeAsync(ScalarId(raw), 100, Ct); await client.WriteValueAsync(ScalarId(raw), 73, Ct); await Wait(() => seen.Any(v => Equals(Clr(v), 73))); var v = seen.Last(v => Equals(Clr(v), 73)); Require(StatusCode.IsGood(v.StatusCode) && SourceTime(v) > Epoch && ServerTime(v) > Epoch, "Bad monitor quality or missing timestamps"); Observations.Add(new { sourceTimestamp = SourceTime(v).ToString("O"), serverTimestamp = ServerTime(v).ToString("O"), status = v.StatusCode.ToString() }); await client.UnsubscribeAsync(ScalarId(raw), Ct); }); + await Case("event-filter-and-native-ack", async () => { var alarms = new ConcurrentQueue(); client.AlarmEvent += (_, a) => alarms.Enqueue(a); await client.SubscribeAlarmsAsync(ObjectIds.Server, 100, Ct); await Wait(() => alarms.Any(a => a.Message == "matrix-alarm" && a.ActiveState && a.Severity == 800)); var alarm = alarms.Last(a => a.Message == "matrix-alarm"); Require(alarm.EventId?.Length > 0, "Missing event ID"); Require(StatusCode.IsGood(await client.AcknowledgeAlarmAsync(new NodeId(Alarm, raw).ToString(), alarm.EventId!, "matrix-ack", Ct)), "Ack refused"); await Wait(() => alarms.Any(a => a.Message == "matrix-alarm" && a.AckedState)); await client.UnsubscribeAlarmsAsync(Ct); }); + await Case("raw-history-CP-paging", async () => { var values = await client.HistoryReadRawAsync(ScalarId(raw), Epoch, Epoch.AddMinutes(1), 100, Ct); Require(values.Count == 25, "Expected 25 raw samples, got " + values.Count); Require(values.Select(v => Convert.ToInt32(Clr(v))).SequenceEqual(Enumerable.Range(0, 25)), "Raw page loss/duplicates/order"); }); + await Case("event-history-CP-paging", async () => + { + var filter = new EventFilter(); +#if SDK2 + filter.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId")) { AttributeId = Attributes.Value }; +#else + filter.SelectClauses.Add(new SimpleAttributeOperand {TypeDefinitionId=ObjectTypeIds.BaseEventType,BrowsePath=new QualifiedNameCollection {new QualifiedName("EventId")},AttributeId=Attributes.Value}); +#endif + var events = await client.HistoryReadEventsAsync(new NodeId(Folder, raw), Epoch, Epoch.AddMinutes(1), filter, 100, Ct); Require(events.Count == 25, "Expected 25 event samples, got " + events.Count); +#if SDK2 + var ids = events.Select(e => System.Text.Encoding.UTF8.GetString((byte[])ZB.MOM.WW.OtOpcUa.Client.Shared.Helpers.ValueConverter.ToClrObject(e.Fields[0])!)); +#else + var ids=events.Select(e=>System.Text.Encoding.UTF8.GetString((byte[])e.Fields[0].Value)); +#endif + Require(ids.SequenceEqual(Enumerable.Range(0, 25).Select(i => $"e{i:00}")), "Event history loss/duplicates/order across timestamp ties"); + }); + await Case("event-history-where-filter-positive-and-negative", async () => + { + foreach (var type in new[] { ObjectTypeIds.BaseEventType, ObjectTypeIds.AuditEventType }) + { + var filter = new EventFilter(); +#if SDK2 + filter.SelectClauses += new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId")) { AttributeId = Attributes.Value }; + filter.WhereClause.Push(FilterOperator.OfType, Variant.From(type)); +#else + filter.SelectClauses.Add(new SimpleAttributeOperand(ObjectTypeIds.BaseEventType,new QualifiedName("EventId")){AttributeId=Attributes.Value}); + filter.WhereClause.Push(FilterOperator.OfType,new LiteralOperand(new Variant(type))); +#endif + var events = await client.HistoryReadEventsAsync(new NodeId(Folder, raw), Epoch, Epoch.AddMinutes(1), filter, 100, Ct); + Require(events.Count == (type == ObjectTypeIds.BaseEventType ? 25 : 0), $"Where filter for {type} expected {(type == ObjectTypeIds.BaseEventType ? 25 : 0)} got {events.Count}"); + } + }); + var unset = await client.ReadValueAsync(new NodeId(Folder + "/UnsetTimestamp", raw), Ct); + Observations.Add(new { sourceTimestamp = SourceTime(unset).ToString("O"), serverTimestamp = ServerTime(unset).ToString("O"), timestampObservation = "Fixture source supplied DateTime.MinValue (unset). These are unmodified application/SDK wire observations; no historian algorithm or expected timestamp was changed." }); + await Benchmark(client, ScalarId(raw), VectorId(raw)); + await client.DisconnectAsync(Ct); + } + static OpcUaClientDriverOptions DriverOptions(string endpoint) => new() + { + EndpointUrl = endpoint, + SecurityPolicy = OpcUaSecurityPolicy.Basic256Sha256, + SecurityMode = OpcUaSecurityMode.SignAndEncrypt, + AuthType = OpcUaAuthType.Username, + Username = "apikey", + Password = Password, + AutoAcceptCertificates = false, + RawTags = [new("scalar",JsonSerializer.Serialize(new {nodeId=$"nsu={RawUri};s={Scalar}"}),true), + new("vector",JsonSerializer.Serialize(new {nodeId=$"nsu={RawUri};s={Vector}"}),true), + new("uns",JsonSerializer.Serialize(new {nodeId=$"nsu={UnsUri};s=Eq/Scalar"}),true)], + SessionName = "MatrixDriver", + ApplicationUri = "urn:localhost:MatrixDriver" + }; + static string IsolatedLocalData() + { + var local = Path.GetFullPath(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)); + Require(local.StartsWith(Path.GetFullPath(Root) + Path.DirectorySeparatorChar, StringComparison.Ordinal), "Driver/CLI canonical PKI must be in task-owned root: " + local); + return local; + } + static async Task PrepareDriver() + { + var local = IsolatedLocalData(); + await using var driver = new OpcUaClientDriver(DriverOptions("opc.tcp://127.0.0.1:1"), "matrix"); + // The production configuration method creates the application's own certificate without + // attempting a session. Reflection is confined to preparation; all wire operations below + // call the unchanged public driver API. + var type = typeof(OpcUaClientDriver); + var hook = Activator.CreateInstance(type.GetNestedType("CertValidationHook", BindingFlags.NonPublic)!, true)!; + await (Task)type.GetMethod("BuildApplicationConfigurationAsync", BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(driver, [hook, Ct])!; + File.WriteAllText(Path.Combine(Root, "driver-pki.txt"), Path.Combine(local, "OtOpcUa", "pki")); + } + static async Task Driver(string endpoint) + { + IsolatedLocalData(); + await using var driver = new OpcUaClientDriver(DriverOptions(endpoint), "matrix"); + await Case("driver-trusted-encrypted-API-key-connect", () => driver.InitializeAsync("{}", Ct)); + await Case("driver-raw-UNS-URI-scalar-and-array-read", async () => + { + var values = await driver.ReadAsync(["scalar", "vector", "uns"], Ct); + Require(values.Count == 3 && values.All(x => x.StatusCode == 0), "Read failure"); + Require(values[0].Value is int && values[1].Value is int[] && values[2].Value is int, "Lost scalar/array type"); + }); + await Case("driver-scalar-array-write-roundtrip", async () => + { + var writes = await driver.WriteAsync([new WriteRequest("scalar", 91), new WriteRequest("vector", new[] { 11, 12, 13 })], Ct); + Require(writes.All(x => x.StatusCode == 0), "Write failed"); + var values = await driver.ReadAsync(["scalar", "vector"], Ct); + Require(Equals(values[0].Value, 91) && ((int[])values[1].Value!).SequenceEqual(new[] { 11, 12, 13 }), "Write round trip mismatch"); + }); + await Case("driver-monitor-quality-timestamps", async () => + { + var seen = new ConcurrentQueue(); + driver.OnDataChange += (_, e) => seen.Enqueue(e.Snapshot); + var subscription = await driver.SubscribeAsync(["scalar"], TimeSpan.FromMilliseconds(100), Ct); + await driver.WriteAsync([new WriteRequest("scalar", 92)], Ct); + await Wait(() => seen.Any(x => Equals(x.Value, 92))); + var value = seen.Last(x => Equals(x.Value, 92)); + Require(value.StatusCode == 0 && value.SourceTimestampUtc.HasValue && value.ServerTimestampUtc > Epoch, "Missing monitor quality/timestamps"); + Observations.Add(new { value.SourceTimestampUtc, value.ServerTimestampUtc, value.StatusCode }); + await driver.UnsubscribeAsync(subscription, Ct); + }); + Observations.Add(new { capability = "driver-history", status = "not-applicable", evidence = "Driver has no IHistoryProvider/HistoryRead capability; .claude/rules/drivers.md #655. Shared/CLI own history." }); + await driver.ShutdownAsync(Ct); + } + static async Task Performance(string endpoint) + { + using var client = new OpcUaClientService(); + var info = await client.ConnectAsync(Settings(endpoint), Ct); + Require(info.SecurityMode == "SignAndEncrypt" && info.SecurityPolicyUri == SecurityPolicies.Basic256Sha256, "Unexpected benchmark endpoint policy"); + var namespaces = (string[])Clr(await client.ReadValueAsync(VariableIds.Server_NamespaceArray, Ct))!; + var raw = (ushort)Array.IndexOf(namespaces, RawUri); + Require(raw > 0, "Missing benchmark raw namespace"); + Observations.Add(new {info.SecurityMode, info.SecurityPolicyUri, tieredCompilation=Environment.GetEnvironmentVariable("DOTNET_TieredCompilation"), scope="Two scalar/Int32[3] variables; zero data monitors during measurement; five fixture variables and one native condition"}); + await Case("warmed-production-client-read-write-measurement", () => Benchmark(client, new NodeId(Scalar, raw), new NodeId(Vector, raw))); + await client.DisconnectAsync(Ct); + } + static async Task Benchmark(OpcUaClientService client, NodeId scalar, NodeId vector) + { + // Local diagnostic subset; excludes reconnect, history and load. + var rounds = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_ROUNDS") ?? "3"); + var warmups = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_WARMUPS") ?? "10"); + var measured = int.Parse(Environment.GetEnvironmentVariable("OPCUA_MATRIX_PERF_MEASURED") ?? "50"); + Require(rounds > 0 && warmups >= 0 && measured > 0, "Invalid performance sample counts"); + for (int round = 1; round <= rounds; round++) + { + for (int warmup = 0; warmup < warmups; warmup++) + { + await client.ReadValueAsync(scalar, Ct); await client.WriteValueAsync(scalar, warmup, Ct); + await client.ReadValueAsync(vector, Ct); await client.WriteValueAsync(vector, new[] { warmup, warmup + 1, warmup + 2 }, Ct); + } + var samples = new Dictionary>(); + var before = System.Diagnostics.Process.GetCurrentProcess(); var cpu = before.TotalProcessorTime; + var allocated = GC.GetTotalAllocatedBytes(); var gc = new[] { GC.CollectionCount(0), GC.CollectionCount(1), GC.CollectionCount(2) }; + foreach (var operation in new[] { "scalar-read", "scalar-write", "array-read", "array-write" }) + { + var times = new List(); + for (int i = 0; i < measured; i++) + { + var start = System.Diagnostics.Stopwatch.GetTimestamp(); + if (operation == "scalar-read") await client.ReadValueAsync(scalar, Ct); + else if (operation == "scalar-write") Require(StatusCode.IsGood(await client.WriteValueAsync(scalar, i, Ct)), "Benchmark scalar write"); + else if (operation == "array-read") await client.ReadValueAsync(vector, Ct); + else Require(StatusCode.IsGood(await client.WriteValueAsync(vector, new[] { i, i + 1, i + 2 }, Ct)), "Benchmark array write"); + times.Add(System.Diagnostics.Stopwatch.GetElapsedTime(start).TotalMilliseconds); + } + samples[operation] = times; + } + before.Refresh(); + Observations.Add(new + { + benchmarkRound = round, + warmupOperationsPerKind = 10, + operationsPerKind = 50, + latencyMilliseconds = samples.ToDictionary(x => x.Key, x => new { p50 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.50) - 1), p95 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.95) - 1), p99 = x.Value.Order().ElementAt((int)Math.Ceiling(measured * 0.99) - 1), throughputOperationsPerSecond = measured * 1000.0 / x.Value.Sum() }), + clientCpuMilliseconds = (before.TotalProcessorTime - cpu).TotalMilliseconds, + clientAllocatedBytes = GC.GetTotalAllocatedBytes() - allocated, clientAllocatedBytesPerOperation = (GC.GetTotalAllocatedBytes() - allocated) / (4.0 * measured), + clientWorkingSetBytes = before.WorkingSet64, + clientPrivateBytes = before.PrivateMemorySize64, + clientThreads = before.Threads.Count, + clientHandles = before.HandleCount, + collections = new[] { GC.CollectionCount(0) - gc[0], GC.CollectionCount(1) - gc[1], GC.CollectionCount(2) - gc[2] } + }); + } + } + sealed class Auth : IOpcUaUserAuthenticator + { + public Task AuthenticateUserNameAsync(string username, string password, CancellationToken ct) => Task.FromResult(username == "apikey" && password == Password ? OpcUaUserAuthResult.AllowKey("matrix", [OpcUaDataPlaneRoles.WriteOperate, OpcUaDataPlaneRoles.AlarmAck], "matrix") : OpcUaUserAuthResult.Deny("Invalid ephemeral fixture key")); + } + sealed class Gateway : IOpcUaNodeWriteGateway { public Task WriteAsync(string id, object? value, AddressSpaceRealm realm, CancellationToken ct) => Task.FromResult(new NodeWriteOutcome(true, null)); } + sealed class Source(string root) : IHistorianDataSource + { + void Log(string operation, int rows) => File.AppendAllText(Path.Combine(root, "history-requests.jsonl"), JsonSerializer.Serialize(new { operation, rows }) + "\n"); + public int MaxRawValuesPerRead => 7; + public Task ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) + { var all = Enumerable.Range(0, 25).Select(i => new DataValueSnapshot(i, 0, Epoch.AddSeconds(i), Epoch.AddSeconds(i))).Where(v => v.SourceTimestampUtc >= s && v.SourceTimestampUtc < e).ToList(); var rows = all.Take((int)(m == 0 ? 7 : m)).ToArray(); Log("raw", rows.Length); return Task.FromResult(new CoreHistory(rows, all.Count > rows.Length ? [1] : null)); } + public Task ReadEventsAsync(string? source, DateTime s, DateTime e, int max, CancellationToken ct) + { var all = Enumerable.Range(0, 25).Select(i => new HistoricalEvent($"e{i:00}", "matrix", Epoch.AddSeconds(i / 2), Epoch, "matrix-history", 500)).Where(v => v.EventTimeUtc >= s && v.EventTimeUtc < e).ToList(); var rows = all.Take(Math.Min(7, max <= 0 ? 7 : max)).ToArray(); Log("events", rows.Length); return Task.FromResult(new HistoricalEventsResult(rows, all.Count > rows.Length ? [1] : null)); } + public Task ReadProcessedAsync(string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) => NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct); + public Task ReadAtTimeAsync(string r, IReadOnlyList t, CancellationToken ct) => NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct); + public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot(); + public void Dispose() { } + } +} diff --git a/scripts/validation/opcua-sdk-compatibility/README.md b/scripts/validation/opcua-sdk-compatibility/README.md new file mode 100644 index 00000000..6b3ea2ea --- /dev/null +++ b/scripts/validation/opcua-sdk-compatibility/README.md @@ -0,0 +1,96 @@ +# OPC UA SDK process compatibility + +Run from a dedicated candidate checkout with .NET 10, Python 3, PowerShell 7, +and Docker. Supply the unchanged original application's existing `host/` and +`client/` publish directories. The output directory must not already exist. + +```powershell +./scripts/e2e/test-opcua-sdk-compatibility.ps1 ` + -BaselineRoot /absolute/original-checkout ` + -BaselineArtifacts /absolute/original-publishes ` + -CandidateRoot /absolute/dedicated-candidate-checkout ` + -OutputDirectory /absolute/fresh-evidence-directory +``` + +The baseline console references the exact published production assemblies; +it does not rebuild or edit the original application. The candidate console +references the production host, Client.Shared and OPC UA driver projects, and +the candidate CLI is separately published. Different SDK versions always run +in different processes. Source revisions, SDK assemblies, SHA-256 hashes, +commands, per-case results, actual namespace URIs, negotiated Shared security, +loaded client SDK assemblies, timestamps and backend request logs are retained. + +The required Shared legs are original→candidate, candidate→original and +candidate→candidate. A fourth original→original leg provides a reference for +both the application contract and the small performance sample. Every Shared +leg checks the exact scalar, array and condition browse identities, both +namespaces by URI, typed scalar/Int32[3] read/write, monitored Good quality and +source/server timestamps, the production alarm event filter and native +acknowledgment, and complete raw/event history across continuation points. +Event-history results include timestamp ties, exact IDs/order and positive and +negative `OfType` predicates. The historian is a deterministic fixture behind +the production server's session-bound continuation store; these results do +not certify any SQL Server, QuestDB or AVEVA backend. + +Server, Shared client and driver require trusted Basic256Sha256 +SignAndEncrypt peers, with certificate autoaccept disabled. Public certificates +are exchanged before sessions open. The test authenticator accepts one exact, +random, API-key-shaped credential from the process environment; production +ACL, write and condition-method checks use the resulting key identity. This +fixture does not replace cryptographic API-key-authenticator coverage. + +The driver and actual CLI run in a disposable `--network none` Linux container, +alongside a separate baseline or candidate server process. PKI lives only in +the mounted task directory; no host profile PKI or fleet endpoint is touched. +The driver hardcodes `LocalApplicationData/OtOpcUa/pki`, so Linux's task-local +`XDG_DATA_HOME` supplies isolation. The helper refuses a driver store outside +its task root. CLI uses its own task-local canonical store. CLI currently +forces `AutoAcceptCertificates=true` and exposes no strict-validation flag +(`CommandBase.CreateConnectionSettings`); the exchanged peers and encrypted +server remain in use, but CLI is not counted as strict client validation. +Driver history is N/A because the application removed that capability (#655); +Shared and CLI execute history reads instead. + +Three measured rounds follow ten warmup operations per kind in each round: +50 scalar reads, scalar writes, array reads and array writes per kind. The +sample records nearest-rank p50/p95/p99, sequential throughput, client CPU, +allocated bytes, GC collections, working set, private bytes, threads and +handles. This is a small local diagnostic on a shared development host. It +has no production fan-out, reconnect, queue, backpressure, backend or soak +performance claim and does not establish Gate G4. + +A separately published `server` helper can be used by other local consumer +checks. Supply a fresh directory and `OPCUA_MATRIX_PASSWORD` in the process +environment, then launch `dotnet ProcessHost.dll server `. +`ready.json`/`endpoint.txt` identify the ephemeral loopback port; touching +`/stop` shuts it down. Its default is strict SignAndEncrypt only. +`OPCUA_MATRIX_SERVER_SECURITY=none` is an explicit fixture-only override for +checking an existing unsecured consumer subset; the matrix runner clears that +override. Anonymous is disabled in either mode. Secure clients must place +their public certificates under `server-pki/trusted/certs` before server start +and trust the resulting `server-pki/own/certs` certificate. + +Explicitly supplied timestamps are asserted. Unset source timestamps are +recorded separately as observations, with their original application/SDK wire +values. The harness does not change historian algorithms or normalize the +unset-timestamp sentinel to manufacture parity. + +For the supplementary sustained warmup experiment, use the built consoles from +one completed matrix run: + +```sh +python3 scripts/validation/opcua-sdk-compatibility/performance.py \ + --baseline-harness /absolute/run/build-baseline/bin/Release/net10.0/ProcessHost.dll \ + --candidate-harness /absolute/run/build-candidate/bin/Release/net10.0/ProcessHost.dll \ + --output /absolute/fresh-performance-directory +``` + +This runs three fresh baseline/candidate process pairs serially, with 1,000 +warmup and 2,000 measured operations per kind in each process. Both server and +client receive `DOTNET_TieredCompilation=0` for this experiment only; the +application's shipped configuration is unchanged. The recorded budget is a +15% increase in the median of the three p95 samples for any operation kind. +Exit code 2 means a diagnostic budget regression, while process failures +remain errors. Even an experiment within that budget does not certify the +complete production performance matrix. The allocation-per-operation field +is the average across all four operation kinds in a measured workload. diff --git a/scripts/validation/opcua-sdk-compatibility/container-leg.sh b/scripts/validation/opcua-sdk-compatibility/container-leg.sh new file mode 100644 index 00000000..00a8b4fc --- /dev/null +++ b/scripts/validation/opcua-sdk-compatibility/container-leg.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Runs only inside the disposable, network-isolated task container. +set -euo pipefail +mkdir -p /task/driver/data /task/cli/data /task/server +XDG_DATA_HOME=/task/driver/data dotnet /candidate/ProcessHost.dll prepare-driver /task/driver > /task/driver-prepare.log 2>&1 +OPCUA_MATRIX_CLIENT_PKI=/task/cli/data/OtOpcUaClient/pki XDG_DATA_HOME=/task/cli/data dotnet /candidate/ProcessHost.dll prepare /task/cli > /task/cli-prepare.log 2>&1 +mkdir -p /task/server/server-pki/trusted/certs +cp /task/driver/data/OtOpcUa/pki/own/certs/* /task/server/server-pki/trusted/certs/ +cp /task/cli/data/OtOpcUaClient/pki/own/certs/* /task/server/server-pki/trusted/certs/ +dotnet /server/ProcessHost.dll server /task/server > /task/server.log 2>&1 & +server_pid=$! +cleanup() { + touch /task/server/stop + wait "$server_pid" || true +} +trap cleanup EXIT +for attempt in $(seq 1 600); do + test ! -f /task/server/ready.json || break + kill -0 "$server_pid" || exit 1 + sleep 0.1 +done +test -f /task/server/ready.json +endpoint=$(cat /task/server/endpoint.txt) +mkdir -p /task/driver/data/OtOpcUa/pki/trusted/certs /task/cli/data/OtOpcUaClient/pki/trusted/certs +cp /task/server/server-pki/own/certs/* /task/driver/data/OtOpcUa/pki/trusted/certs/ +cp /task/server/server-pki/own/certs/* /task/cli/data/OtOpcUaClient/pki/trusted/certs/ +result=0 +XDG_DATA_HOME=/task/driver/data dotnet /candidate/ProcessHost.dll driver /task/driver "$endpoint" > /task/driver.log 2>&1 || result=1 +# CLI currently autoaccepts by product design; it has no strict-validation flag. +# Its public certificates are nevertheless explicitly exchanged in this fixture. +XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll read --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev/Scalar' > /task/cli-read.log 2>&1 || result=1 +XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll historyread --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev/Scalar' --start 2026-10-01T00:00:00Z --end 2026-10-01T00:01:00Z --max 100 > /task/cli-raw-history.log 2>&1 || result=1 +XDG_DATA_HOME=/task/cli/data dotnet /cli/ZB.MOM.WW.OtOpcUa.Client.CLI.dll historyread --url "$endpoint" --security signandencrypt --username apikey --password "$OPCUA_MATRIX_PASSWORD" --node 'ns=2;s=Matrix/dev' --start 2026-10-01T00:00:00Z --end 2026-10-01T00:01:00Z --max 100 --events --fields EventId,Message > /task/cli-event-history.log 2>&1 || result=1 +exit "$result" diff --git a/scripts/validation/opcua-sdk-compatibility/performance.py b/scripts/validation/opcua-sdk-compatibility/performance.py new file mode 100644 index 00000000..0188d74d --- /dev/null +++ b/scripts/validation/opcua-sdk-compatibility/performance.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Supplementary warmed, matched native-process experiment; not full Gate G4.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import shutil +import statistics +import subprocess +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline-harness', type=Path, required=True) + parser.add_argument('--candidate-harness', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--tiered-compilation', choices=['0', '1'], default='0') + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32), + DOTNET_TieredCompilation=args.tiered_compilation, OPCUA_MATRIX_PERF_ROUNDS='1', + OPCUA_MATRIX_PERF_WARMUPS='1000', OPCUA_MATRIX_PERF_MEASURED='2000') + for key in ('OPCUA_MATRIX_SERVER_SECURITY', 'OPCUA_MATRIX_CLIENT_PKI'): + env.pop(key, None) + metadata = {'scope': __doc__, 'environment': {k: env[k] for k in ('DOTNET_TieredCompilation', 'OPCUA_MATRIX_PERF_ROUNDS', 'OPCUA_MATRIX_PERF_WARMUPS', 'OPCUA_MATRIX_PERF_MEASURED')}, + 'dotnetSdk': subprocess.check_output(['dotnet', '--version'], text=True).strip(), 'p95BudgetPercent': 15, + 'dateUtc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'logicalCpuCount': os.cpu_count()} + commands, results, hashes = [], [], [] + + def write(path, data): + path.write_text(json.dumps(data, indent=2) + '\n') + + def run(command, log): + commands.append({'argv': list(map(str, command)), 'log': str(log)}) + with log.open('w') as stream: + result = subprocess.run(list(map(str, command)), stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=240) + if result.returncode: + raise RuntimeError(f'Process exited {result.returncode}; {log}') + + def copy_certs(source, destination): + destination.mkdir(parents=True, exist_ok=True) + files = list(source.glob('*')) + if not files: + raise RuntimeError('Missing public certificates: ' + str(source)) + for path in files: + shutil.copy2(path, destination / path.name) + + try: + for pair in range(1, 4): + for version, harness in [('baseline', args.baseline_harness.resolve()), ('candidate', args.candidate_harness.resolve())]: + leg = output / f'pair-{pair}-{version}' + leg.mkdir() + client, server = leg / 'client', leg / 'server' + client.mkdir() + server.mkdir() + run(['dotnet', harness, 'prepare', client], leg / 'prepare.log') + copy_certs(client / 'client-pki/own/certs', server / 'server-pki/trusted/certs') + server_log = (leg / 'server.log').open('w') + command = ['dotnet', str(harness), 'server', str(server)] + commands.append({'argv': command, 'log': str(leg / 'server.log')}) + process = subprocess.Popen(command, env=env, stdout=server_log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + 60 + while not (server / 'ready.json').exists(): + if process.poll() is not None or time.monotonic() > deadline: + raise RuntimeError('Server failed to start: ' + str(leg / 'server.log')) + time.sleep(.1) + ready = json.loads((server / 'ready.json').read_text()) + copy_certs(server / 'server-pki/own/certs', client / 'client-pki/trusted/certs') + run(['dotnet', harness, 'performance', client, ready['endpoint']], leg / 'client.log') + result = json.loads((client / 'result.json').read_text()) + result.update({'pair': pair, 'version': version, 'server': ready}) + finally: + (server / 'stop').touch() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=10) + server_log.close() + result['serverResources'] = json.loads((server / 'server-resources.json').read_text()) + results.append(result) + write(leg / 'measurement.json', result) + summary = [] + for operation in ('scalar-read', 'scalar-write', 'array-read', 'array-write'): + samples = {} + for version in ('baseline', 'candidate'): + rounds = [o for r in results if r['version'] == version for o in r['observations'] if 'benchmarkRound' in o] + samples[version] = {'p95Milliseconds': [r['latencyMilliseconds'][operation]['p95'] for r in rounds], + 'allocatedBytesPerOperation': [r['clientAllocatedBytesPerOperation'] for r in rounds], + 'clientWorkingSetBytes': [r['clientWorkingSetBytes'] for r in rounds], + 'serverWorkingSetBytes': [r['serverResources']['WorkingSet64'] for r in results if r['version'] == version]} + ratio = statistics.median(samples['candidate']['p95Milliseconds']) / statistics.median(samples['baseline']['p95Milliseconds']) + summary.append({'operation': operation, 'samples': samples, 'medianP95ChangePercent': 100 * (ratio - 1), 'status': 'diagnostic-regression' if ratio > 1.15 else 'diagnostic-within-budget'}) + metadata['summary'] = summary + metadata['status'] = 'diagnostic-regression' if any(r['status'] == 'diagnostic-regression' for r in summary) else 'diagnostic-within-budget' + for row in summary: + print(f"{row['operation']}: median p95 {row['medianP95ChangePercent']:+.1f}% ({row['status']})") + finally: + for harness in (args.baseline_harness.resolve(), args.candidate_harness.resolve()): + for path in [harness, *harness.parent.glob('Opc.Ua*.dll'), *harness.parent.glob('ZB.MOM.WW.OtOpcUa*.dll'), *harness.parent.glob('OtOpcUa.Host.dll')]: + hashes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'bytes': path.stat().st_size}) + write(output / 'artifact-hashes.json', hashes) + write(output / 'commands.json', commands) + write(output / 'measurements.json', results) + write(output / 'summary.json', metadata) + return 2 if metadata.get('status') == 'diagnostic-regression' else 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/validation/opcua-sdk-compatibility/run.py b/scripts/validation/opcua-sdk-compatibility/run.py new file mode 100644 index 00000000..5a32f1ac --- /dev/null +++ b/scripts/validation/opcua-sdk-compatibility/run.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Separate-process production application compatibility gate (no mixed SDK process).""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import secrets +import shutil +import subprocess +import time +import uuid +import xml.etree.ElementTree as ET + +HERE = Path(__file__).resolve().parent +HOST = 'src/Server/ZB.MOM.WW.OtOpcUa.Host/ZB.MOM.WW.OtOpcUa.Host.csproj' +CLIENT = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.Shared/ZB.MOM.WW.OtOpcUa.Client.Shared.csproj' +DRIVER = 'src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient/ZB.MOM.WW.OtOpcUa.Driver.OpcUaClient.csproj' +CLI = 'src/Client/ZB.MOM.WW.OtOpcUa.Client.CLI/ZB.MOM.WW.OtOpcUa.Client.CLI.csproj' + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + '\n') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--baseline-root', required=True, type=Path) + parser.add_argument('--baseline-artifacts', required=True, type=Path, help='Existing immutable host/ and client/ publishes') + parser.add_argument('--candidate-root', required=True, type=Path, help='Dedicated candidate checkout; this script builds it') + parser.add_argument('--container-image', default='mcr.microsoft.com/dotnet/sdk:10.0.103-noble', help='Disposable local driver/CLI isolation; must supply .NET 10 and bash') + parser.add_argument('--output', required=True, type=Path, help='Fresh nonexistent evidence directory') + args = parser.parse_args() + args.baseline_root = args.baseline_root.resolve() + args.candidate_root = args.candidate_root.resolve() + args.baseline_artifacts = args.baseline_artifacts.resolve() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=False) + source_info = {} + for name, root in [('baseline', args.baseline_root), ('candidate', args.candidate_root)]: + source_info[name] = {'root': str(root), 'commit': subprocess.check_output(['git', '-C', str(root), 'rev-parse', 'HEAD'], text=True).strip(), + 'sdkPackages': str(root / 'Directory.Packages.props')} + source_info['scope'] = 'Loopback application contract and 3 warmed 50-operation rounds per scalar/Int32[3] read/write kind; shared development host, no full load/reconnect/failover/backend/soak performance claim.' + source_info['dateUtc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + write_json(args.output / 'source-info.json', source_info) + commands = [] + matrix = [] + env = dict(os.environ, OPCUA_MATRIX_PASSWORD='otua_matrix_' + secrets.token_urlsafe(32)) + env.pop('OPCUA_MATRIX_CLIENT_PKI', None) + env.pop('OPCUA_MATRIX_SERVER_SECURITY', None) + + def run(command, log, process_env=env, timeout=300): + commands.append({'argv': ['' if x == env['OPCUA_MATRIX_PASSWORD'] else str(x) for x in command], 'log': str(log)}) + with log.open('w') as stream: + result = subprocess.run([str(x) for x in command], stdout=stream, stderr=subprocess.STDOUT, env=process_env, timeout=timeout) + return result.returncode + + def build(version): + folder = args.output / ('build-' + version) + folder.mkdir() + project = ET.Element('Project', Sdk='Microsoft.NET.Sdk') + props = ET.SubElement(project, 'PropertyGroup') + for k, v in {'OutputType': 'Exe', 'TargetFramework': 'net10.0', 'Nullable': 'enable', 'ImplicitUsings': 'enable', 'LangVersion': 'latest', 'DefineConstants': 'SDK2' if version == 'candidate' else ''}.items(): + ET.SubElement(props, k).text = v + items = ET.SubElement(project, 'ItemGroup') + ET.SubElement(items, 'FrameworkReference', Include='Microsoft.AspNetCore.App') + if version == 'candidate': + for path in (HOST, CLIENT, DRIVER): + ET.SubElement(items, 'ProjectReference', Include=str(args.candidate_root / path)) + else: + dlls = {} + for path in ('host', 'client'): + for dll in (args.baseline_artifacts / path).glob('*.dll'): + dlls[dll.name] = dll + for name, dll in sorted(dlls.items()): + ref = ET.SubElement(items, 'Reference', Include=name[:-4]) + ET.SubElement(ref, 'HintPath').text = str(dll) + ET.SubElement(ref, 'Private').text = 'true' + ET.SubElement(items, 'Compile', Include=str(HERE / 'ProcessHost.cs')) + csproj = folder / 'ProcessHost.csproj' + ET.ElementTree(project).write(csproj, encoding='unicode') + status = run(['dotnet', 'build', csproj, '-c', 'Release', '-m:1', '--nologo'], folder / 'build.log') + if status: + raise RuntimeError(f'{version} harness build failed ({status}); see {folder / "build.log"}') + return folder / 'bin/Release/net10.0/ProcessHost.dll' + + def certificates(source, target): + target.mkdir(parents=True, exist_ok=True) + files = list(source.glob('*')) + if not files: + raise RuntimeError('No public certificates at ' + str(source)) + for file in files: + shutil.copy2(file, target / file.name) + + def receipt(paths): + return [{'path': str(p), 'sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'bytes': p.stat().st_size} for p in sorted(set(paths)) if p.is_file()] + + baseline = candidate = candidate_cli = None + try: + # Baseline builds only the portable harness against the exact published production DLLs. + baseline = build('baseline') + candidate = build('candidate') + candidate_cli = args.output / 'candidate-cli' + status = run(['dotnet', 'publish', args.candidate_root / CLI, '-c', 'Release', '-o', candidate_cli, '-m:1', '--nologo'], args.output / 'candidate-cli-build.log') + if status: + raise RuntimeError('Candidate CLI publish failed') + for name, client, server in [('baseline-client-to-baseline-server', baseline, baseline), ('baseline-client-to-candidate-server', baseline, candidate), ('candidate-client-to-baseline-server', candidate, baseline), ('candidate-client-to-candidate-server', candidate, candidate)]: + leg = args.output / name + leg.mkdir() + server_root = leg / 'server' + client_root = leg / 'shared' + server_root.mkdir() + client_root.mkdir() + status = run(['dotnet', client, 'prepare', client_root], leg / 'prepare.log') + if status: + matrix.append({'name': name, 'status': 'failed', 'stage': 'client-certificate-preparation'}) + continue + certificates(client_root / 'client-pki/own/certs', server_root / 'server-pki/trusted/certs') + server_log = (leg / 'server.log').open('w') + command = ['dotnet', str(server), 'server', str(server_root)] + commands.append({'argv': command, 'log': str(leg / 'server.log')}) + proc = subprocess.Popen(command, stdout=server_log, stderr=subprocess.STDOUT, env=env) + try: + deadline = time.monotonic() + 60 + while not (server_root / 'ready.json').exists(): + if proc.poll() is not None or time.monotonic() > deadline: + raise RuntimeError(f'{name}: server did not become ready; see server.log') + time.sleep(.1) + ready = json.loads((server_root / 'ready.json').read_text()) + certificates(server_root / 'server-pki/own/certs', client_root / 'client-pki/trusted/certs') + status = run(['dotnet', client, 'client', client_root, ready['endpoint']], leg / 'shared.log') + result = json.loads((client_root / 'result.json').read_text()) + history = [json.loads(x) for x in (server_root / 'history-requests.jsonl').read_text().splitlines()] if (server_root / 'history-requests.jsonl').exists() else [] + result.update({'name': name, 'server': ready, 'status': 'passed' if status == 0 else 'failed', 'historyBackendRequests': history, 'certificateValidation': 'explicit trusted peers; server and Shared client autoaccept disabled', 'authentication': 'ephemeral API-key-shaped credential, exact fixture authenticator; real authority/ACL/method gates'}) + write_json(leg / 'leg.json', result) + matrix.append(result) + except Exception as exc: + matrix.append({'name': name, 'status': 'failed', 'error': str(exc)}) + finally: + (server_root / 'stop').touch() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.terminate() + proc.wait(timeout=10) + server_log.close() + image_info = subprocess.check_output(['docker', 'image', 'inspect', args.container_image, '--format', '{{json .}}'], text=True) + image_data = json.loads(image_info) + write_json(args.output / 'container-image.json', {k: image_data.get(k) for k in ('Id', 'RepoDigests', 'Architecture', 'Os')}) + for server_name, server in [('baseline', baseline), ('candidate', candidate)]: + name = 'candidate-driver-cli-to-' + server_name + '-server' + leg = args.output / name + leg.mkdir() + shutil.copy2(HERE / 'container-leg.sh', leg / 'run.sh') + container_name = 'otopcua-sdk2-matrix-' + uuid.uuid4().hex[:12] + command = ['docker', 'run', '--rm', '--name', container_name, '--network', 'none', '-e', 'OPCUA_MATRIX_PASSWORD', + '-v', str(candidate.parent) + ':/candidate:ro', '-v', str(server.parent) + ':/server:ro', + '-v', str(candidate_cli) + ':/cli:ro', '-v', str(leg) + ':/task', args.container_image, 'bash', '/task/run.sh'] + try: + status = run(command, leg / 'container.log') + finally: + subprocess.run(['docker', 'rm', '-f', container_name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if (leg / 'driver/result.json').exists(): + result = json.loads((leg / 'driver/result.json').read_text()) + else: + result = {'cases': [], 'error': 'Driver result missing; inspect preparation/container/server logs'} + for operation, expected in [('read', 'Value: 92'), ('raw-history', '25 values returned.'), ('event-history', '25 events returned.')]: + log = leg / ('cli-' + operation + '.log') + passed = log.exists() and expected in log.read_text() + result['cases'].append({'name': 'production-CLI-' + operation, 'status': 'passed' if passed else 'failed', 'log': str(log), 'expectedOutput': expected}) + result.update({'name': name, 'status': 'passed' if status == 0 and all(x['status'] == 'passed' for x in result['cases']) else 'failed', + 'container': container_name, 'network': 'none; all peers in task container, separate SDK processes', + 'driverCertificateValidation': 'explicit trusted peers; autoaccept disabled', 'cliCertificateValidation': 'explicit exchanged peers; CLI always configures AutoAcceptCertificates=true (CommandBase.CreateConnectionSettings)', + 'server': json.loads((leg / 'server/ready.json').read_text()) if (leg / 'server/ready.json').exists() else None}) + write_json(leg / 'leg.json', result) + matrix.append(result) + + finally: + paths = [p for p in (baseline, candidate, args.baseline_root / 'Directory.Packages.props', args.candidate_root / 'Directory.Packages.props', HERE / 'ProcessHost.cs', HERE / 'run.py', HERE / 'container-leg.sh') if p is not None] + for folder in (args.baseline_artifacts / 'host', args.baseline_artifacts / 'client', candidate.parent if candidate else None, candidate_cli): + if folder is not None: + paths.extend(folder.glob('Opc.Ua*.dll')) + paths.extend(folder.glob('*.deps.json')) + paths.extend(folder.glob('ZB.MOM.WW.OtOpcUa*.dll')) + paths.extend(folder.glob('OtOpcUa.Host.dll')) + paths.extend(folder.glob('*.runtimeconfig.json')) + write_json(args.output / 'artifact-hashes.json', receipt(paths)) + write_json(args.output / 'commands.json', commands) + write_json(args.output / 'matrix.json', matrix) + return 1 if not matrix or any(x['status'] != 'passed' for x in matrix) else 0 + + +if __name__ == '__main__': + raise SystemExit(main())