Joseph Doherty
997c78df8a
The one failure on record (48 s vs 3 s, load 140-280) was a bare 'Condition not met within 00:00:45'. It did not reproduce here (0/10 alone and 0/6 full Host integration runs under 28 CPU burners at load ~300), so no timing was changed; the timeout now carries the deployment row (status, artifact size, failure reason), every per-node state (status, started/applied, failure) vs the expected node ids, and both nodes' cluster views - enough to tell a slow apply from a node that never got the dispatch from a failed apply. Co-Authored-By: Claude <noreply@anthropic.com>
299 lines
17 KiB
C#
299 lines
17 KiB
C#
using Microsoft.EntityFrameworkCore;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
|
||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||
using ZB.MOM.WW.OtOpcUa.Configuration.Validation;
|
||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||
|
||
/// <summary>
|
||
/// End-to-end deploy of a UNS folder hierarchy through the <b>real</b> <c>ConfigComposer</c>: seed a
|
||
/// 1-area / 1-line / 1-equipment UNS plus a v3 raw-tag chain (RawFolder → DriverInstance → Device →
|
||
/// Tag) + a <see cref="UnsTagReference"/> projecting the raw tag into the equipment, <c>StartDeployment</c>,
|
||
/// then assert the deployment's persisted artifact decodes (via
|
||
/// <see cref="DeploymentArtifact.ParseComposition"/>).
|
||
/// <para>
|
||
/// <b>v3 dual namespace (Batch 4):</b> the <c>AddressSpaceComposer</c> un-darkens BOTH subtrees. The
|
||
/// <see cref="Deploying_a_uns_hierarchy_carries_folder_nodes_and_leaves_equipment_tags_dark"/> case pins the
|
||
/// artifact-decode invariant that the retired equipment-namespace <c>EquipmentTags</c> set stays EMPTY
|
||
/// (values now flow through the raw + UNS variable nodes). The
|
||
/// <see cref="Composing_the_seeded_config_emits_the_raw_tag_and_uns_reference_variables"/> case drives the
|
||
/// real <c>AddressSpaceComposer</c> over the SAME seeded config (loaded back from the deploy DB) and pins
|
||
/// the Batch-4 dual-tree: the seeded raw tag surfaces as a Raw-realm Variable keyed by its RawPath, and the
|
||
/// UnsTagReference surfaces as a UNS-realm Variable carrying that RawPath (the Organizes target + fan-out).
|
||
/// <para><i>Note:</i> the composer is what WP1 lights up. WP3 migrated the artifact-decode seam
|
||
/// (<c>DeploymentArtifact.ParseComposition</c>) to carry both realms too, so
|
||
/// <see cref="Deployed_artifact_round_trips_both_namespaces_and_seals_on_the_cluster"/> (added in WP5) pins
|
||
/// the dual tree through the FULL deploy → persisted-artifact → decode round-trip a driver node applies, and
|
||
/// additionally asserts the dual-namespace deploy seals across the redundant 2-node cluster.</para>
|
||
/// </para>
|
||
/// <para>
|
||
/// The OPC UA address-space browse is exercised separately against a real SDK node manager in
|
||
/// <c>AddressSpaceApplierHierarchyTests</c>, because the in-process <see cref="TwoNodeClusterHarness"/>
|
||
/// binds the no-op address-space sink.
|
||
/// </para>
|
||
/// </summary>
|
||
public sealed class EquipmentNamespaceMaterializationTests
|
||
{
|
||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||
|
||
// Equipment.EquipmentId must equal DraftValidator.DeriveEquipmentId(EquipmentUuid) — the
|
||
// 'EquipmentIdNotDerived' rule (EquipmentId = 'EQ-' + first 12 hex of the UUID). A fixed UUID
|
||
// keeps the derived id stable for the assertions below.
|
||
private static readonly Guid EquipmentUuid = Guid.Parse("11111111-1111-1111-1111-111111111111");
|
||
private static readonly string EquipmentId = DraftValidator.DeriveEquipmentId(EquipmentUuid); // EQ-111111111111
|
||
|
||
/// <summary>Verifies a deployed UNS hierarchy carries its Area/Line/Equipment folder nodes into the
|
||
/// composed artifact with their friendly UNS names, and — the v3 Batch-1 DARK contract — that the
|
||
/// equipment-tag plan set is EMPTY even though a raw tag was seeded (raw-tag variables are Batch 4).</summary>
|
||
[Fact]
|
||
public async Task Deploying_a_uns_hierarchy_carries_folder_nodes_and_leaves_equipment_tags_dark()
|
||
{
|
||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||
|
||
// Seed the parent ServerCluster + ClusterNode rows the SQL FKs require (the c1 config below
|
||
// FKs to ServerCluster; on the in-memory provider this is unnecessary — FKs aren't enforced).
|
||
await harness.SeedDefaultClusterAsync("c1");
|
||
await SeedUnsHierarchyAsync(harness);
|
||
|
||
await using var scope = harness.NodeA.Services.CreateAsyncScope();
|
||
var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();
|
||
|
||
var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
|
||
result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}");
|
||
var deploymentId = result.DeploymentId!.Value.Value;
|
||
|
||
// The artifact is composed + persisted at dispatch time; assert on it without waiting for
|
||
// Seal (sealing depends on driver-node acks, orthogonal to composition correctness).
|
||
var artifact = Array.Empty<byte>();
|
||
await WaitForAsync(async () =>
|
||
{
|
||
await using var db = await CreateDbAsync(harness);
|
||
var d = await db.Deployments.AsNoTracking()
|
||
.FirstOrDefaultAsync(x => x.DeploymentId == deploymentId, Ct);
|
||
if (d is { ArtifactBlob.Length: > 0 })
|
||
{
|
||
artifact = d.ArtifactBlob;
|
||
return true;
|
||
}
|
||
return false;
|
||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||
|
||
var composition = DeploymentArtifact.ParseComposition(artifact);
|
||
|
||
// The UNS folder hierarchy decodes with its friendly UNS names (browse names), not MachineCodes.
|
||
composition.EquipmentNodes.ShouldContain(e => e.EquipmentId == EquipmentId && e.DisplayName == "station-1");
|
||
composition.UnsAreas.ShouldContain(a => a.UnsAreaId == "nw-area-filling" && a.DisplayName == "filling");
|
||
composition.UnsLines.ShouldContain(l => l.UnsLineId == "nw-line-1" && l.DisplayName == "line-1");
|
||
|
||
// v3 DARK contract: no equipment-tag variable plans materialize this batch — even though a raw
|
||
// tag ("Speed" / FullName 40001) was seeded under the device. Batch 4 lights these via the UNS↔Raw fan-out.
|
||
composition.EquipmentTags.ShouldBeEmpty();
|
||
}
|
||
|
||
/// <summary>BATCH-4 dual namespace: the real <see cref="AddressSpaceComposer"/> over the seeded config
|
||
/// (loaded back from the deploy DB) emits BOTH subtrees — the seeded raw tag as a Raw-realm Variable
|
||
/// keyed by its RawPath, and the UnsTagReference as a UNS-realm Variable carrying that RawPath (the
|
||
/// Organizes UNS→Raw target + fan-out source) and inheriting the raw tag's DataType. Drives the composer
|
||
/// directly (not the artifact-decode round-trip, which WP3 migrates).</summary>
|
||
[Fact]
|
||
public async Task Composing_the_seeded_config_emits_the_raw_tag_and_uns_reference_variables()
|
||
{
|
||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||
await harness.SeedDefaultClusterAsync("c1");
|
||
await SeedUnsHierarchyAsync(harness);
|
||
|
||
await using var db = await CreateDbAsync(harness);
|
||
var composition = AddressSpaceComposer.Compose(
|
||
await db.UnsAreas.AsNoTracking().ToListAsync(Ct),
|
||
await db.UnsLines.AsNoTracking().ToListAsync(Ct),
|
||
await db.Equipment.AsNoTracking().ToListAsync(Ct),
|
||
await db.DriverInstances.AsNoTracking().ToListAsync(Ct),
|
||
await db.ScriptedAlarms.AsNoTracking().ToListAsync(Ct),
|
||
unsTagReferences: await db.UnsTagReferences.AsNoTracking().ToListAsync(Ct),
|
||
tags: await db.Tags.AsNoTracking().ToListAsync(Ct),
|
||
rawFolders: await db.RawFolders.AsNoTracking().ToListAsync(Ct),
|
||
devices: await db.Devices.AsNoTracking().ToListAsync(Ct),
|
||
tagGroups: await db.TagGroups.AsNoTracking().ToListAsync(Ct));
|
||
|
||
// Raw subtree: the seeded raw tag is a Raw-realm Variable keyed by its RawPath (Plant/Modbus/dev-1/Speed).
|
||
var rawTag = composition.RawTags.ShouldHaveSingleItem();
|
||
rawTag.TagId.ShouldBe("tag-speed");
|
||
rawTag.Realm.ShouldBe(AddressSpaceRealm.Raw);
|
||
rawTag.NodeId.ShouldBe("Plant/Modbus/dev-1/Speed");
|
||
rawTag.DataType.ShouldBe("Float");
|
||
|
||
// UNS subtree: the UnsTagReference is a UNS-realm Variable carrying the backing RawPath + inherited type.
|
||
var unsVar = composition.UnsReferenceVariables.ShouldHaveSingleItem();
|
||
unsVar.Realm.ShouldBe(AddressSpaceRealm.Uns);
|
||
unsVar.EquipmentId.ShouldBe(EquipmentId);
|
||
unsVar.NodeId.ShouldBe("filling/line-1/station-1/Speed");
|
||
unsVar.EffectiveName.ShouldBe("Speed");
|
||
unsVar.BackingRawPath.ShouldBe("Plant/Modbus/dev-1/Speed"); // Organizes UNS→Raw + fan-out
|
||
unsVar.DataType.ShouldBe("Float");
|
||
|
||
// The retired equipment-namespace tag path stays empty (values flow through raw + UNS now).
|
||
composition.EquipmentTags.ShouldBeEmpty();
|
||
}
|
||
|
||
/// <summary>BATCH-4 harness round-trip + redundancy: deploying the seeded dual-namespace config through the
|
||
/// REAL deploy pipeline (AdminOperations → ConfigPublishCoordinator → BOTH DriverHostActors) seals on the
|
||
/// 2-node cluster, AND the persisted artifact round-trips BOTH realms via
|
||
/// <see cref="DeploymentArtifact.ParseComposition(System.ReadOnlySpan{byte})"/> — the raw tag as a Raw-realm
|
||
/// Variable keyed by its RawPath, and the UnsTagReference as a UNS-realm Variable carrying that RawPath (the
|
||
/// Organizes/fan-out backing path). This is the harness-level dual-namespace materialization proof: the
|
||
/// second namespace neither breaks redundant sealing nor is lost across artifact serialization. (WP3 migrated
|
||
/// the artifact-decode seam to carry both realms, so — unlike the composer-direct test above — this exercises
|
||
/// the full deploy → persisted-artifact → decode path a driver node actually applies.)</summary>
|
||
[Fact]
|
||
public async Task Deployed_artifact_round_trips_both_namespaces_and_seals_on_the_cluster()
|
||
{
|
||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||
await harness.SeedDefaultClusterAsync("c1");
|
||
await SeedUnsHierarchyAsync(harness);
|
||
|
||
await using var scope = harness.NodeA.Services.CreateAsyncScope();
|
||
var client = scope.ServiceProvider.GetRequiredService<IAdminOperationsClient>();
|
||
|
||
var result = await client.StartDeploymentAsync(createdBy: "alice@test", Ct);
|
||
result.Outcome.ShouldBe(StartDeploymentOutcome.Accepted, $"Deploy not accepted: {result.Message}");
|
||
var deploymentId = result.DeploymentId!.Value.Value;
|
||
|
||
// Redundancy non-interference: the dual-namespace deploy seals after BOTH DriverHostActors apply — the
|
||
// second namespace does not break the redundant seal path (the ServiceLevel / primary-gate machinery
|
||
// is orthogonal to the address-space namespace count; see FailoverDuringDeploy / PrimaryGateFailover).
|
||
var artifact = Array.Empty<byte>();
|
||
await WaitForAsync(async () =>
|
||
{
|
||
await using var db = await CreateDbAsync(harness);
|
||
var d = await db.Deployments.AsNoTracking()
|
||
.FirstOrDefaultAsync(x => x.DeploymentId == deploymentId, Ct);
|
||
if (d is { Status: DeploymentStatus.Sealed, ArtifactBlob.Length: > 0 })
|
||
{
|
||
artifact = d.ArtifactBlob;
|
||
return true;
|
||
}
|
||
return false;
|
||
}, TwoNodeClusterHarness.DeploySealTimeout, () => DescribeDeploymentAsync(harness, deploymentId));
|
||
|
||
await using (var db = await CreateDbAsync(harness))
|
||
{
|
||
var nodeStates = await db.NodeDeploymentStates.AsNoTracking()
|
||
.Where(s => s.DeploymentId == deploymentId)
|
||
.ToListAsync(Ct);
|
||
nodeStates.Count.ShouldBe(2, "both cluster nodes record a per-node deployment state");
|
||
nodeStates.ShouldAllBe(s => s.Status == NodeDeploymentStatus.Applied);
|
||
}
|
||
|
||
// Round-trip the PERSISTED artifact through the artifact-decode seam and pin the dual tree.
|
||
var composition = DeploymentArtifact.ParseComposition(artifact);
|
||
|
||
// Raw subtree: the seeded raw tag survives as a Raw-realm Variable keyed by its RawPath.
|
||
var rawTag = composition.RawTags.ShouldHaveSingleItem();
|
||
rawTag.TagId.ShouldBe("tag-speed");
|
||
rawTag.Realm.ShouldBe(AddressSpaceRealm.Raw);
|
||
rawTag.NodeId.ShouldBe("Plant/Modbus/dev-1/Speed");
|
||
|
||
// UNS subtree: the UnsTagReference survives as a UNS-realm Variable carrying the backing RawPath (the
|
||
// Organizes UNS→Raw target + the fan-out source).
|
||
var unsVar = composition.UnsReferenceVariables.ShouldHaveSingleItem();
|
||
unsVar.Realm.ShouldBe(AddressSpaceRealm.Uns);
|
||
unsVar.EquipmentId.ShouldBe(EquipmentId);
|
||
unsVar.NodeId.ShouldBe("filling/line-1/station-1/Speed");
|
||
unsVar.BackingRawPath.ShouldBe("Plant/Modbus/dev-1/Speed");
|
||
|
||
// The retired equipment-namespace tag path stays empty (values flow through raw + UNS now).
|
||
composition.EquipmentTags.ShouldBeEmpty();
|
||
}
|
||
|
||
private static async Task SeedUnsHierarchyAsync(TwoNodeClusterHarness harness)
|
||
{
|
||
await using var db = await CreateDbAsync(harness);
|
||
|
||
// v3 raw-tag chain: RawFolder → DriverInstance(RawFolderId) → Device(DriverInstanceId) →
|
||
// Tag(DeviceId). The Namespace entity is deleted; equipment no longer binds a driver/device.
|
||
db.RawFolders.Add(new RawFolder { RawFolderId = "raw-plant", ClusterId = "c1", Name = "Plant" });
|
||
// Disabled so the driver-role nodes don't spawn a live Modbus driver (no endpoint to reach);
|
||
// the composition still carries the instance (ParseComposition does not filter on Enabled).
|
||
db.DriverInstances.Add(new DriverInstance
|
||
{
|
||
DriverInstanceId = "drv-modbus", ClusterId = "c1", RawFolderId = "raw-plant",
|
||
Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", Enabled = false,
|
||
});
|
||
db.Devices.Add(new Device
|
||
{
|
||
DeviceId = "dev-1", DriverInstanceId = "drv-modbus",
|
||
Name = "dev-1", DeviceConfig = "{}",
|
||
});
|
||
db.UnsAreas.Add(new UnsArea { UnsAreaId = "nw-area-filling", ClusterId = "c1", Name = "filling" });
|
||
db.UnsLines.Add(new UnsLine { UnsLineId = "nw-line-1", UnsAreaId = "nw-area-filling", Name = "line-1" });
|
||
db.Equipment.Add(new Equipment
|
||
{
|
||
EquipmentId = EquipmentId, EquipmentUuid = EquipmentUuid,
|
||
UnsLineId = "nw-line-1", Name = "station-1", MachineCode = "STATION_001",
|
||
});
|
||
db.Tags.Add(new Tag
|
||
{
|
||
TagId = "tag-speed", DeviceId = "dev-1",
|
||
Name = "Speed", DataType = "Float", AccessLevel = TagAccessLevel.Read,
|
||
TagConfig = "{\"FullName\":\"40001\"}",
|
||
});
|
||
// v3 Batch 4: the equipment projects the raw tag by reference (reference-only UNS).
|
||
db.UnsTagReferences.Add(new UnsTagReference
|
||
{
|
||
UnsTagReferenceId = "ref-speed", EquipmentId = EquipmentId, TagId = "tag-speed",
|
||
});
|
||
|
||
await db.SaveChangesAsync(Ct);
|
||
}
|
||
|
||
private static async Task<OtOpcUaConfigDbContext> CreateDbAsync(TwoNodeClusterHarness harness)
|
||
{
|
||
var factory = harness.NodeA.Services.GetRequiredService<IDbContextFactory<OtOpcUaConfigDbContext>>();
|
||
return await factory.CreateDbContextAsync();
|
||
}
|
||
|
||
private static async Task WaitForAsync(Func<Task<bool>> condition, TimeSpan timeout,
|
||
Func<Task<string>>? diagnose = null)
|
||
{
|
||
var deadline = DateTime.UtcNow + timeout;
|
||
while (DateTime.UtcNow < deadline)
|
||
{
|
||
if (await condition()) return;
|
||
await Task.Delay(200);
|
||
}
|
||
throw new TimeoutException($"Condition not met within {timeout}"
|
||
+ (diagnose is null ? "" : $"; state at timeout: {await diagnose()}"));
|
||
}
|
||
|
||
/// <summary>
|
||
/// lmxopcua#843: the seal wait used to time out with a bare "Condition not met within 45s" (seen once at load
|
||
/// average 140–280, 48 s against a normal 3 s). Report where the deploy stood — the deployment row, each node's
|
||
/// per-node state, and both nodes' cluster views — so the next failure says whether it was a slow apply, a node
|
||
/// that never received the dispatch (no ack, still Applying), or a failed apply.
|
||
/// </summary>
|
||
private static async Task<string> DescribeDeploymentAsync(TwoNodeClusterHarness harness, Guid deploymentId)
|
||
{
|
||
await using var db = await CreateDbAsync(harness);
|
||
var d = await db.Deployments.AsNoTracking().FirstOrDefaultAsync(x => x.DeploymentId == deploymentId);
|
||
var nodes = await db.NodeDeploymentStates.AsNoTracking()
|
||
.Where(s => s.DeploymentId == deploymentId).ToListAsync();
|
||
static string View(Akka.Actor.ActorSystem s) => string.Join(",",
|
||
Akka.Cluster.Cluster.Get(s).State.Members.Select(m => $"{m.Address.Port}:{m.Status}"))
|
||
+ $" unreachable={Akka.Cluster.Cluster.Get(s).State.Unreachable.Count}";
|
||
return $"deployment={d?.Status.ToString() ?? "<missing>"} artifact={d?.ArtifactBlob.Length ?? 0}B "
|
||
+ $"failure='{d?.FailureReason}' created={d?.CreatedAtUtc:O} now={DateTime.UtcNow:O}; "
|
||
+ $"nodes=[{string.Join("; ", nodes.Select(n => $"{n.NodeId} {n.Status} started={n.StartedAtUtc:O} applied={n.AppliedAtUtc:O} failure='{n.FailureReason}'"))}] "
|
||
+ $"(expected {harness.NodeANodeId}, {harness.NodeBNodeId}); "
|
||
+ $"A view=[{View(harness.NodeASystem)}] B view=[{View(harness.NodeBSystem)}]";
|
||
}
|
||
}
|