Merge R2-04 Failure-visibility trio (arch-review round 2) [PR #433]
Findings 01/S-1 (AddressSpaceApplyOutcome failure field + apply.failed logging, no optimistic success), 06/S-1 (Galaxy write fails closed -> BadCommunicationError + #5 revert, no knowingly-lost raw Write), 03/S4 (PrimaryGatePolicy default-deny unknown-role-multi-driver on all gates + scripted-alarm emit gate). T13/T15 (2-node live gates) deferred -- T15 is the behavior-affecting S4 live gate, must run in heavy pass. Clean merge, build clean.
This commit is contained in:
Joseph Doherty
+211
@@ -0,0 +1,211 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// archreview 06/S-1: Galaxy writes must fail CLOSED. When <c>AdviseSupervisory</c> fails, the raw
|
||||
/// Write (which the file comment notes "doesn't throw but the value never reaches the galaxy") must
|
||||
/// NOT be issued — the writer returns <c>BadCommunicationError</c> so the #5 node-revert fires instead
|
||||
/// of leaving a phantom-Good node. These tests drive the real write pipeline through the internal
|
||||
/// <see cref="IMxWriteOps"/> seam (the SDK session types are sealed + internal-ctor and cannot be faked).
|
||||
/// </summary>
|
||||
public sealed class GatewayGalaxyDataWriterFailClosedTests
|
||||
{
|
||||
private static GatewayGalaxyDataWriter NewWriter()
|
||||
=> new(new GalaxyMxSession(new Config.GalaxyMxAccessOptions(ClientName: "OtOpcUa-Test")), writeUserId: 0);
|
||||
|
||||
// ---------------- T5: seam ----------------
|
||||
|
||||
/// <summary>A secured-write classification routes through WriteSecured and never advises (regression via the seam).</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_SecuredClassification_RoutesThroughWriteSecured_NoAdvise()
|
||||
{
|
||||
var ops = new FakeMxWriteOps();
|
||||
var writer = NewWriter();
|
||||
|
||||
var result = await writer.WriteOneForTestAsync(
|
||||
ops, new WriteRequest("Tag.Sec", true), SecurityClassification.SecuredWrite, CancellationToken.None);
|
||||
|
||||
result.StatusCode.ShouldBe(StatusCodeMap.Good);
|
||||
ops.Invokes.Count(r => r.Command.Kind == MxCommandKind.WriteSecured).ShouldBe(1);
|
||||
ops.Invokes.Any(r => r.Command.Kind == MxCommandKind.AdviseSupervisory).ShouldBeFalse();
|
||||
ops.RawWriteHandles.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
// ---------------- T6: fail-closed on advise failure ----------------
|
||||
|
||||
/// <summary>When AdviseSupervisory fails (non-OK protocol status), the raw Write must NOT be issued and
|
||||
/// the writer returns Bad (which fires the #5 node revert). The handle must not linger in the supervised
|
||||
/// cache so the next write retries the advise.</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_AdviseFails_ReturnsBadCommunicationError_AndNeverIssuesRawWrite()
|
||||
{
|
||||
var ops = new FakeMxWriteOps
|
||||
{
|
||||
InvokeResponder = _ => new MxCommandReply
|
||||
{
|
||||
ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.MxaccessFailure, Message = "advise boom" },
|
||||
},
|
||||
};
|
||||
var writer = NewWriter();
|
||||
|
||||
var result = await writer.WriteOneForTestAsync(
|
||||
ops, new WriteRequest("Tag.A", true), SecurityClassification.FreeAccess, CancellationToken.None);
|
||||
|
||||
result.StatusCode.ShouldBe(StatusCodeMap.BadCommunicationError);
|
||||
ops.Invokes.Count(r => r.Command.Kind == MxCommandKind.AdviseSupervisory).ShouldBe(1);
|
||||
ops.RawWriteHandles.ShouldBeEmpty(); // the knowingly-lost raw write was NOT issued
|
||||
writer.CachedSupervisedHandleCount.ShouldBe(0); // advise failure forgot the handle → next write retries
|
||||
}
|
||||
|
||||
/// <summary>When advise succeeds, the raw Write proceeds and a clean reply returns Good (regression).</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_AdviseSucceeds_RawWriteProceeds()
|
||||
{
|
||||
var ops = new FakeMxWriteOps
|
||||
{
|
||||
InvokeResponder = _ => new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } },
|
||||
};
|
||||
var writer = NewWriter();
|
||||
|
||||
var result = await writer.WriteOneForTestAsync(
|
||||
ops, new WriteRequest("Tag.B", 42), SecurityClassification.FreeAccess, CancellationToken.None);
|
||||
|
||||
result.StatusCode.ShouldBe(StatusCodeMap.Good);
|
||||
ops.Invokes.Count(r => r.Command.Kind == MxCommandKind.AdviseSupervisory).ShouldBe(1);
|
||||
ops.RawWriteHandles.ShouldHaveSingleItem(); // the raw write was issued after the successful advise
|
||||
writer.CachedSupervisedHandleCount.ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---------------- T7: counters ----------------
|
||||
|
||||
/// <summary>A fail-closed advise increments galaxy.writes.advise_failed (and not unconfirmed).</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_AdviseFails_Increments_AdviseFailed_Meter()
|
||||
{
|
||||
using var advise = new MeterRecorder("galaxy.writes.advise_failed");
|
||||
using var unconfirmed = new MeterRecorder("galaxy.writes.unconfirmed");
|
||||
var ops = new FakeMxWriteOps
|
||||
{
|
||||
InvokeResponder = _ => new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.MxaccessFailure } },
|
||||
};
|
||||
var writer = NewWriter();
|
||||
|
||||
await writer.WriteOneForTestAsync(ops, new WriteRequest("Tag.A", true), SecurityClassification.FreeAccess, CancellationToken.None);
|
||||
|
||||
advise.Total.ShouldBe(1);
|
||||
unconfirmed.Total.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>An advised write whose reply carries an EMPTY statuses array returns Good but increments
|
||||
/// galaxy.writes.unconfirmed (the commit is unconfirmed pending gateway WriteComplete correlation).</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_EmptyStatuses_ReturnsGood_And_Increments_Unconfirmed_Meter()
|
||||
{
|
||||
using var advise = new MeterRecorder("galaxy.writes.advise_failed");
|
||||
using var unconfirmed = new MeterRecorder("galaxy.writes.unconfirmed");
|
||||
var ops = new FakeMxWriteOps
|
||||
{
|
||||
InvokeResponder = _ => new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } },
|
||||
RawWriteResponder = () => new MxCommandReply(), // no ProtocolStatus, EMPTY statuses ⇒ Good
|
||||
};
|
||||
var writer = NewWriter();
|
||||
|
||||
var result = await writer.WriteOneForTestAsync(ops, new WriteRequest("Tag.B", 1), SecurityClassification.FreeAccess, CancellationToken.None);
|
||||
|
||||
result.StatusCode.ShouldBe(StatusCodeMap.Good);
|
||||
unconfirmed.Total.ShouldBe(1);
|
||||
advise.Total.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>A reply carrying an MX status ROW increments neither meter (not empty-statuses, advise OK).</summary>
|
||||
[Fact]
|
||||
public async Task WriteOne_StatusRowReply_Increments_Neither_Meter()
|
||||
{
|
||||
using var advise = new MeterRecorder("galaxy.writes.advise_failed");
|
||||
using var unconfirmed = new MeterRecorder("galaxy.writes.unconfirmed");
|
||||
var okStatusReply = new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } };
|
||||
okStatusReply.Statuses.Add(new MxStatusProxy { Success = 1, Category = MxStatusCategory.Ok });
|
||||
var ops = new FakeMxWriteOps
|
||||
{
|
||||
InvokeResponder = _ => new MxCommandReply { ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok } },
|
||||
RawWriteResponder = () => okStatusReply,
|
||||
};
|
||||
var writer = NewWriter();
|
||||
|
||||
var result = await writer.WriteOneForTestAsync(ops, new WriteRequest("Tag.C", 2), SecurityClassification.FreeAccess, CancellationToken.None);
|
||||
|
||||
result.StatusCode.ShouldBe(StatusCodeMap.Good); // Success/Ok status row ⇒ Good
|
||||
advise.Total.ShouldBe(0);
|
||||
unconfirmed.Total.ShouldBe(0); // NOT empty-statuses ⇒ not metered unconfirmed
|
||||
}
|
||||
|
||||
// ---------------- fakes ----------------
|
||||
|
||||
/// <summary>Listens to a single instrument by name on the Galaxy driver meter and sums the values.</summary>
|
||||
private sealed class MeterRecorder : IDisposable
|
||||
{
|
||||
private readonly string _name;
|
||||
private readonly System.Diagnostics.Metrics.MeterListener _listener;
|
||||
private long _total;
|
||||
private readonly object _gate = new();
|
||||
|
||||
public MeterRecorder(string instrumentName)
|
||||
{
|
||||
_name = instrumentName;
|
||||
_listener = new System.Diagnostics.Metrics.MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == EventPump.MeterName && instrument.Name == _name)
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
_listener.SetMeasurementEventCallback<long>((_, value, _, _) => { lock (_gate) _total += value; });
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
public long Total { get { lock (_gate) return _total; } }
|
||||
|
||||
public void Dispose() => _listener.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>A fake <see cref="IMxWriteOps"/> that records every call and returns canned replies.</summary>
|
||||
private sealed class FakeMxWriteOps : IMxWriteOps
|
||||
{
|
||||
public string SessionId => "sess-1";
|
||||
public int ServerHandle => 100;
|
||||
public int NextHandle { get; init; } = 7;
|
||||
|
||||
public List<string> AddItemRefs { get; } = new();
|
||||
public List<MxCommandRequest> Invokes { get; } = new();
|
||||
public List<int> RawWriteHandles { get; } = new();
|
||||
|
||||
/// <summary>Responder for <see cref="InvokeAsync"/>; null ⇒ an OK (empty) reply.</summary>
|
||||
public Func<MxCommandRequest, MxCommandReply>? InvokeResponder { get; init; }
|
||||
/// <summary>Responder for <see cref="WriteRawAsync"/>; null ⇒ an empty (Good) reply.</summary>
|
||||
public Func<MxCommandReply>? RawWriteResponder { get; init; }
|
||||
|
||||
public Task<int> AddItemAsync(string fullRef, CancellationToken ct)
|
||||
{
|
||||
AddItemRefs.Add(fullRef);
|
||||
return Task.FromResult(NextHandle);
|
||||
}
|
||||
|
||||
public Task<MxCommandReply> InvokeAsync(MxCommandRequest request, CancellationToken ct)
|
||||
{
|
||||
Invokes.Add(request);
|
||||
return Task.FromResult(InvokeResponder?.Invoke(request) ?? new MxCommandReply());
|
||||
}
|
||||
|
||||
public Task<MxCommandReply> WriteRawAsync(int itemHandle, MxValue value, int userId, CancellationToken ct)
|
||||
{
|
||||
RawWriteHandles.Add(itemHandle);
|
||||
return Task.FromResult(RawWriteResponder?.Invoke() ?? new MxCommandReply());
|
||||
}
|
||||
}
|
||||
}
|
||||
+28
@@ -173,6 +173,34 @@ public sealed class GatewayGalaxyLiveReopenAndWriteTests
|
||||
$"borrow smoke: subscribed {WriteRef} -> handle {match.ItemHandle}; borrowed write Good with AddItemCallCount=0; control AddItemCallCount=1");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// archreview 06/S-1 guard — after the fail-closed advise change, a NORMAL advised no-login write
|
||||
/// must STILL return Good (0). Guards against over-eager fail-closing: the advise round-trip succeeds
|
||||
/// on a healthy gateway, so the raw Write proceeds and commits. If this ever returns Bad on a healthy
|
||||
/// gateway, the fail-closed branch is mis-judging a successful advise. Skip-gated like the others.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Live_normal_advised_write_still_returns_good_under_failclosed_advise()
|
||||
{
|
||||
var (endpoint, apiKey) = RequireLiveGatewayOrSkip();
|
||||
var ct = TestContext.Current.CancellationToken;
|
||||
var clientOptions = BuildClientOptions(endpoint, apiKey);
|
||||
|
||||
await using var session = new GalaxyMxSession(new GalaxyMxAccessOptions(ClientName: "OtOpcUaFailClosedGuard"));
|
||||
await session.ConnectAsync(clientOptions, ct);
|
||||
var writer = new GatewayGalaxyDataWriter(session, writeUserId: 0);
|
||||
|
||||
var result = await writer.WriteAsync(
|
||||
[new WriteRequest(WriteRef, 5151.0f)], _ => SecurityClassification.FreeAccess, ct);
|
||||
|
||||
result.ShouldHaveSingleItem().StatusCode.ShouldBe(0u,
|
||||
"a normal advised write must still return Good after the fail-closed advise change (no over-eager fail-close)");
|
||||
writer.CachedSupervisedHandleCount.ShouldBe(1, "the healthy advise should have supervised the handle");
|
||||
|
||||
TestContext.Current.SendDiagnosticMessage(
|
||||
$"fail-closed guard smoke: normal advised write {WriteRef}=5151 returned Good; handle supervised");
|
||||
}
|
||||
|
||||
private static (string Endpoint, string ApiKey) RequireLiveGatewayOrSkip()
|
||||
{
|
||||
var endpoint = Environment.GetEnvironmentVariable("MXGW_ENDPOINT");
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Hosting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// archreview 03/S4 delivery-path guard — the "unit tests can't catch delivery" half. Proves that the
|
||||
/// REAL DistributedPubSub-delivered redundancy snapshot (not a test-injected <c>RedundancyStateChanged</c>)
|
||||
/// actually drives the Primary write gate end-to-end across a live 2-node Akka cluster: after both nodes
|
||||
/// join and the snapshot converges, exactly ONE node's <see cref="DriverHostActor"/> rejects a
|
||||
/// <see cref="DriverHostActor.RouteNodeWrite"/> with <c>"not primary"</c> (the Secondary) while the other
|
||||
/// passes the gate (the Primary — its rejection, if any, is about node MAPPING, never the gate).
|
||||
///
|
||||
/// <para>Before the snapshot delivers, BOTH nodes have an unknown role on a 2-driver cluster and so both
|
||||
/// default-DENY with <c>"not primary (role unknown)"</c> (archreview 03/S4). The poll below therefore
|
||||
/// waits for the delivered snapshot to promote exactly one node to Primary — a broken redundancy-topic
|
||||
/// subscribe (the historical double-break) would leave both stuck "role unknown" and time this out, which
|
||||
/// is exactly the negative control this test provides over the pure-TestKit guards.</para>
|
||||
/// </summary>
|
||||
[Trait("Category", "Failover")]
|
||||
public sealed class PrimaryGateFailoverTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
// A NodeId that is guaranteed to have no driver mapping — so a node that PASSES the primary gate
|
||||
// rejects it with a MAPPING reason ("no driver mapping for node …"), never a gate reason.
|
||||
private const string UnmappedProbeNode = "__pgate_delivery_probe__";
|
||||
|
||||
[Fact]
|
||||
public async Task Delivered_redundancy_snapshot_drives_the_primary_write_gate_across_the_cluster()
|
||||
{
|
||||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||||
|
||||
var driverHostA = harness.NodeA.Services.GetRequiredService<ActorRegistry>().Get<DriverHostActorKey>();
|
||||
var driverHostB = harness.NodeB.Services.GetRequiredService<ActorRegistry>().Get<DriverHostActorKey>();
|
||||
|
||||
// Poll until the DELIVERED snapshot has promoted exactly one node to Primary: one host passes the
|
||||
// gate (reason not "not primary*") and the other is gated ("not primary"). Deadline covers the
|
||||
// ~250ms-debounced initial publish plus cluster convergence margin.
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(45);
|
||||
(bool aGated, bool bGated, string? aReason, string? bReason) last = default;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var a = await RouteProbeWriteAsync(driverHostA);
|
||||
var b = await RouteProbeWriteAsync(driverHostB);
|
||||
last = (IsGateReject(a), IsGateReject(b), a.Reason, b.Reason);
|
||||
|
||||
// Exactly one node gated ⇒ the snapshot converged (one Primary, one Secondary).
|
||||
if (last.aGated ^ last.bGated) break;
|
||||
await Task.Delay(500, Ct);
|
||||
}
|
||||
|
||||
// Exactly one node is the Secondary (gate-rejected); the other is the Primary (passed the gate —
|
||||
// its reject, if any, is a MAPPING reason). This proves the delivered snapshot drives the gate.
|
||||
(last.aGated ^ last.bGated).ShouldBeTrue(
|
||||
$"exactly one node should be gated once the redundancy snapshot converges (A gated={last.aGated} reason='{last.aReason}', B gated={last.bGated} reason='{last.bReason}')");
|
||||
|
||||
var primaryReason = last.aGated ? last.bReason : last.aReason;
|
||||
primaryReason.ShouldNotBeNull();
|
||||
primaryReason!.StartsWith("not primary", StringComparison.Ordinal).ShouldBeFalse(
|
||||
$"the Primary must PASS the gate — its reject reason should be a mapping reason, not '{primaryReason}'");
|
||||
}
|
||||
|
||||
private static async Task<DriverHostActor.NodeWriteResult> RouteProbeWriteAsync(IActorRef driverHost)
|
||||
=> await driverHost.Ask<DriverHostActor.NodeWriteResult>(
|
||||
new DriverHostActor.RouteNodeWrite(UnmappedProbeNode, 0.0), TimeSpan.FromSeconds(10));
|
||||
|
||||
private static bool IsGateReject(DriverHostActor.NodeWriteResult r)
|
||||
=> !r.Success && r.Reason is not null && r.Reason.StartsWith("not primary", StringComparison.Ordinal);
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Guards the failure-visibility surface added for archreview 01/S-1: the applier must report a
|
||||
/// failed rebuild (<see cref="AddressSpaceApplyOutcome.RebuildFailed"/>) and tally swallowed per-node
|
||||
/// sink failures (<see cref="AddressSpaceApplyOutcome.FailedNodes"/> for Apply's own passes; the
|
||||
/// <c>Materialise*</c> passes return their own failed-node counts) instead of reporting optimistic
|
||||
/// success while the running address space is stale or partial.
|
||||
/// </summary>
|
||||
public sealed class AddressSpaceApplierFailureSurfaceTests
|
||||
{
|
||||
// ---------------- T1: RebuildFailed ----------------
|
||||
|
||||
/// <summary>A rebuild whose sink call throws is reported as attempted-but-failed, not optimistic success.</summary>
|
||||
[Fact]
|
||||
public void Apply_WhenRebuildThrows_ReportsRebuildFailed()
|
||||
{
|
||||
var sink = new ConfigurableThrowingSink { ThrowOnRebuild = true };
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
|
||||
var outcome = applier.Apply(AddedEquipmentPlan("new"));
|
||||
|
||||
outcome.RebuildCalled.ShouldBeTrue();
|
||||
outcome.RebuildFailed.ShouldBeTrue();
|
||||
}
|
||||
|
||||
/// <summary>A clean apply reports no failure on either channel — the regression guard for the new fields.</summary>
|
||||
[Fact]
|
||||
public void Apply_HappyPath_ReportsNoFailure()
|
||||
{
|
||||
var sink = new ConfigurableThrowingSink();
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
|
||||
var outcome = applier.Apply(AddedEquipmentPlan("new"));
|
||||
|
||||
outcome.RebuildCalled.ShouldBeTrue();
|
||||
outcome.RebuildFailed.ShouldBeFalse();
|
||||
outcome.FailedNodes.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---------------- T2: FailedNodes (removal-pass tally) ----------------
|
||||
|
||||
/// <summary>A removal-pass alarm-condition write that throws is counted into FailedNodes (swallowed today).</summary>
|
||||
[Fact]
|
||||
public void Apply_WhenRemovalConditionWriteThrows_CountsFailedNodes()
|
||||
{
|
||||
var sink = new ConfigurableThrowingSink { ThrowOnAlarmWrite = true };
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
|
||||
var outcome = applier.Apply(EquipmentRemovalPlan("eq-1"));
|
||||
|
||||
outcome.RemovedNodes.ShouldBe(1);
|
||||
outcome.FailedNodes.ShouldBe(1);
|
||||
}
|
||||
|
||||
// ---------------- T3: Materialise* passes return failed-node counts ----------------
|
||||
|
||||
/// <summary>Every EnsureVariable throwing in MaterialiseEquipmentTags is tallied into the pass's int return.</summary>
|
||||
[Fact]
|
||||
public void MaterialiseEquipmentTags_WhenEnsureVariableThrows_ReturnsFailedCount()
|
||||
{
|
||||
var sink = new ConfigurableThrowingSink { ThrowOnEnsureVariable = true };
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
|
||||
var composition = CompositionWithTags(
|
||||
new EquipmentTagPlan("t1", "eq-1", "drv", FolderPath: "", Name: "A", DataType: "Float", FullName: "40001", Writable: false, Alarm: null),
|
||||
new EquipmentTagPlan("t2", "eq-1", "drv", FolderPath: "", Name: "B", DataType: "Float", FullName: "40002", Writable: false, Alarm: null),
|
||||
new EquipmentTagPlan("t3", "eq-1", "drv", FolderPath: "", Name: "C", DataType: "Float", FullName: "40003", Writable: false, Alarm: null));
|
||||
|
||||
applier.MaterialiseEquipmentTags(composition).ShouldBe(3);
|
||||
}
|
||||
|
||||
/// <summary>All five Materialise* passes return 0 failures on a clean sink — the happy-path regression.</summary>
|
||||
[Fact]
|
||||
public void Materialise_HappyPath_AllPassesReturnZero()
|
||||
{
|
||||
var sink = new ConfigurableThrowingSink();
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
|
||||
var tags = CompositionWithTags(
|
||||
new EquipmentTagPlan("t1", "eq-1", "drv", FolderPath: "", Name: "A", DataType: "Float", FullName: "40001", Writable: false, Alarm: null));
|
||||
var vtags = CompositionWithVirtualTags(
|
||||
new EquipmentVirtualTagPlan("v1", "eq-1", FolderPath: "", Name: "VA", DataType: "Float", Expression: "1", DependencyRefs: Array.Empty<string>(), Historize: false));
|
||||
var alarms = CompositionWithScriptedAlarms(
|
||||
new EquipmentScriptedAlarmPlan("a1", "eq-1", "Alarm", "OffNormalAlarm", 500, "msg", "pred", "true",
|
||||
Array.Empty<string>(), HistorizeToAveva: false, Retain: true, Enabled: true));
|
||||
|
||||
applier.MaterialiseHierarchy(tags).ShouldBe(0);
|
||||
applier.MaterialiseEquipmentTags(tags).ShouldBe(0);
|
||||
applier.MaterialiseEquipmentVirtualTags(vtags).ShouldBe(0);
|
||||
applier.MaterialiseScriptedAlarms(alarms).ShouldBe(0);
|
||||
applier.MaterialiseDiscoveredNodes(
|
||||
"eq-1",
|
||||
Array.Empty<DiscoveredFolder>(),
|
||||
new[] { new DiscoveredVariable("eq-1/D", "eq-1", "D", "Float", Writable: false, IsArray: false, ArrayLength: null) })
|
||||
.ShouldBe(0);
|
||||
}
|
||||
|
||||
// ---------------- fixtures ----------------
|
||||
|
||||
private static AddressSpaceComposition CompositionWithTags(params EquipmentTagPlan[] tags) =>
|
||||
new(Array.Empty<EquipmentNode>(), Array.Empty<DriverInstancePlan>(), Array.Empty<ScriptedAlarmPlan>())
|
||||
{
|
||||
EquipmentTags = tags,
|
||||
};
|
||||
|
||||
private static AddressSpaceComposition CompositionWithVirtualTags(params EquipmentVirtualTagPlan[] vtags) =>
|
||||
new(Array.Empty<EquipmentNode>(), Array.Empty<DriverInstancePlan>(), Array.Empty<ScriptedAlarmPlan>())
|
||||
{
|
||||
EquipmentVirtualTags = vtags,
|
||||
};
|
||||
|
||||
private static AddressSpaceComposition CompositionWithScriptedAlarms(params EquipmentScriptedAlarmPlan[] alarms) =>
|
||||
new(Array.Empty<EquipmentNode>(), Array.Empty<DriverInstancePlan>(), Array.Empty<ScriptedAlarmPlan>())
|
||||
{
|
||||
EquipmentScriptedAlarms = alarms,
|
||||
};
|
||||
|
||||
private static AddressSpacePlan AddedEquipmentPlan(string id) => new(
|
||||
AddedEquipment: new[] { new EquipmentNode(id, id, "line-1") },
|
||||
RemovedEquipment: Array.Empty<EquipmentNode>(),
|
||||
ChangedEquipment: Array.Empty<AddressSpacePlan.EquipmentDelta>(),
|
||||
AddedDrivers: Array.Empty<DriverInstancePlan>(),
|
||||
RemovedDrivers: Array.Empty<DriverInstancePlan>(),
|
||||
ChangedDrivers: Array.Empty<AddressSpacePlan.DriverDelta>(),
|
||||
AddedAlarms: Array.Empty<ScriptedAlarmPlan>(),
|
||||
RemovedAlarms: Array.Empty<ScriptedAlarmPlan>(),
|
||||
ChangedAlarms: Array.Empty<AddressSpacePlan.AlarmDelta>());
|
||||
|
||||
private static AddressSpacePlan EquipmentRemovalPlan(params string[] ids) => new(
|
||||
AddedEquipment: Array.Empty<EquipmentNode>(),
|
||||
RemovedEquipment: ids.Select(id => new EquipmentNode(id, id, "line-1")).ToArray(),
|
||||
ChangedEquipment: Array.Empty<AddressSpacePlan.EquipmentDelta>(),
|
||||
AddedDrivers: Array.Empty<DriverInstancePlan>(),
|
||||
RemovedDrivers: Array.Empty<DriverInstancePlan>(),
|
||||
ChangedDrivers: Array.Empty<AddressSpacePlan.DriverDelta>(),
|
||||
AddedAlarms: Array.Empty<ScriptedAlarmPlan>(),
|
||||
RemovedAlarms: Array.Empty<ScriptedAlarmPlan>(),
|
||||
ChangedAlarms: Array.Empty<AddressSpacePlan.AlarmDelta>());
|
||||
|
||||
/// <summary>An <see cref="IOpcUaAddressSpaceSink"/> whose every sink call can be independently made to
|
||||
/// throw, so tests can drive each Safe* helper's catch branch. Non-throwing calls are no-ops.</summary>
|
||||
private sealed class ConfigurableThrowingSink : IOpcUaAddressSpaceSink
|
||||
{
|
||||
public bool ThrowOnRebuild { get; init; }
|
||||
public bool ThrowOnEnsureVariable { get; init; }
|
||||
public bool ThrowOnEnsureFolder { get; init; }
|
||||
public bool ThrowOnAlarmWrite { get; init; }
|
||||
public bool ThrowOnMaterialiseAlarm { get; init; }
|
||||
|
||||
public void WriteValue(string nodeId, object? value, OpcUaQuality quality, DateTime sourceTimestampUtc) { }
|
||||
|
||||
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime sourceTimestampUtc)
|
||||
{
|
||||
if (ThrowOnAlarmWrite) throw new InvalidOperationException("simulated WriteAlarmCondition fault");
|
||||
}
|
||||
|
||||
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, bool isNative = false)
|
||||
{
|
||||
if (ThrowOnMaterialiseAlarm) throw new InvalidOperationException("simulated MaterialiseAlarmCondition fault");
|
||||
}
|
||||
|
||||
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName)
|
||||
{
|
||||
if (ThrowOnEnsureFolder) throw new InvalidOperationException("simulated EnsureFolder fault");
|
||||
}
|
||||
|
||||
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, string? historianTagname = null, bool isArray = false, uint? arrayLength = null)
|
||||
{
|
||||
if (ThrowOnEnsureVariable) throw new InvalidOperationException("simulated EnsureVariable fault");
|
||||
}
|
||||
|
||||
public void RebuildAddressSpace()
|
||||
{
|
||||
if (ThrowOnRebuild) throw new InvalidOperationException("simulated RebuildAddressSpace fault");
|
||||
}
|
||||
|
||||
public void RaiseNodesAddedModelChange(string affectedNodeId) { }
|
||||
}
|
||||
}
|
||||
+425
@@ -0,0 +1,425 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster.Tools.PublishSubscribe;
|
||||
using Akka.TestKit;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// archreview 03/S4 — the boot-window primary-gate guard. On a MULTI-driver cluster (member count > 1)
|
||||
/// while the role is UNKNOWN, the Primary data-plane gates (inbound write, native ack, native alerts emit)
|
||||
/// default-DENY; on a SINGLE-driver cluster (count <= 1) they default-ALLOW (boot-window / single-node
|
||||
/// posture). Driver member count is injected via the <c>driverMemberCountProvider</c> Props seam.
|
||||
/// </summary>
|
||||
public sealed class DriverHostActorPrimaryGateTests : RuntimeActorTestBase
|
||||
{
|
||||
private static readonly NodeId TestNode = NodeId.Parse("driver-pgate-test");
|
||||
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
|
||||
private static readonly DateTime Ts = new(2026, 7, 13, 10, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
// ---------------- write gate ----------------
|
||||
|
||||
/// <summary>Role unknown + a real driver peer (count 2) ⇒ RouteNodeWrite is DENIED with the
|
||||
/// boot-window reason and the driver sees no write.</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_multi_driver_denies_write_with_role_unknown_reason()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingDriverFactory("Modbus");
|
||||
var dep = SeedWriteTagDeployment(db);
|
||||
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 2);
|
||||
|
||||
var asker = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-1/speed", 123.0), asker.Ref);
|
||||
|
||||
var result = asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
||||
result.Success.ShouldBeFalse();
|
||||
result.Reason.ShouldBe("not primary (role unknown)");
|
||||
recorder.Writes.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>Role unknown + no driver peer (count 1) ⇒ the write is SERVICED (single-node / boot-window
|
||||
/// posture preserved).</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_single_driver_services_write()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingDriverFactory("Modbus");
|
||||
var dep = SeedWriteTagDeployment(db);
|
||||
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 1);
|
||||
|
||||
// The gate must ALLOW (count 1, role unknown). The driver child's async connect can lag the ApplyAck,
|
||||
// so a first write may transiently report "driver not connected" — retry until connected (a real
|
||||
// client retries too). The point under test is that the gate never denies (never "not primary").
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var asker = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-1/speed", 123.0), asker.Ref);
|
||||
var res = asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
||||
res.Reason.ShouldNotBe("not primary");
|
||||
res.Reason.ShouldNotBe("not primary (role unknown)");
|
||||
res.Success.ShouldBeTrue(res.Reason);
|
||||
}, duration: TimeSpan.FromSeconds(10));
|
||||
recorder.Writes.ShouldNotBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>A Primary snapshot services even at count 2; a Secondary snapshot denies with the steady-state
|
||||
/// reason (distinct from the boot-window reason).</summary>
|
||||
[Fact]
|
||||
public void Known_role_wins_over_member_count()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var recorder = new RecordingDriverFactory("Modbus");
|
||||
var dep = SeedWriteTagDeployment(db);
|
||||
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 2);
|
||||
|
||||
// Primary snapshot ⇒ serviced even with a driver peer. Retry through the transient driver-connect
|
||||
// window; the invariant under test is that the gate never denies a Primary.
|
||||
TellRole(actor, RedundancyRole.Primary);
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
var asker1 = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-1/speed", 1.0), asker1.Ref);
|
||||
var res = asker1.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
||||
res.Reason.ShouldNotBe("not primary");
|
||||
res.Success.ShouldBeTrue(res.Reason);
|
||||
}, duration: TimeSpan.FromSeconds(10));
|
||||
|
||||
// Secondary snapshot ⇒ denied with the steady-state reason.
|
||||
TellRole(actor, RedundancyRole.Secondary);
|
||||
var asker2 = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-1/speed", 2.0), asker2.Ref);
|
||||
var denied = asker2.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout);
|
||||
denied.Success.ShouldBeFalse();
|
||||
denied.Reason.ShouldBe("not primary");
|
||||
}
|
||||
|
||||
/// <summary>The denial meter carries the site + reason tags.</summary>
|
||||
[Fact]
|
||||
public void Denied_write_increments_primary_gate_denied_meter_with_tags()
|
||||
{
|
||||
using var recorder = new MeterRecorder("otopcua.redundancy.primary_gate_denied");
|
||||
var db = NewInMemoryDbFactory();
|
||||
var factory = new RecordingDriverFactory("Modbus");
|
||||
var dep = SeedWriteTagDeployment(db);
|
||||
var actor = SpawnWriteHost(db, dep, factory, driverMemberCount: 2);
|
||||
|
||||
var asker = CreateTestProbe();
|
||||
actor.Tell(new DriverHostActor.RouteNodeWrite("eq-1/speed", 9.0), asker.Ref);
|
||||
asker.ExpectMsg<DriverHostActor.NodeWriteResult>(Timeout).Success.ShouldBeFalse();
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
recorder.Total.ShouldBeGreaterThanOrEqualTo(1);
|
||||
recorder.WithTag("site", "write").ShouldBeGreaterThanOrEqualTo(1);
|
||||
recorder.WithTag("reason", "role-unknown").ShouldBeGreaterThanOrEqualTo(1);
|
||||
}, duration: Timeout);
|
||||
}
|
||||
|
||||
// ---------------- ack gate ----------------
|
||||
|
||||
/// <summary>Role unknown + count 2 ⇒ a native-alarm ack is dropped at Warning and the denial meter
|
||||
/// increments with site=ack.</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_multi_driver_drops_native_ack_with_warning_and_meter()
|
||||
{
|
||||
using var meter = new MeterRecorder("otopcua.redundancy.primary_gate_denied");
|
||||
var db = NewInMemoryDbFactory();
|
||||
var dep = SeedAlarmTagDeployment(db);
|
||||
var (actor, _) = SpawnAlarmHost(db, dep, driverMemberCount: 2);
|
||||
|
||||
EventFilter.Warning(contains: "role unknown").ExpectOne(() =>
|
||||
actor.Tell(new DriverHostActor.RouteNativeAlarmAck("eq-1/temp_hi", Comment: null, OperatorUser: "op")));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
meter.WithTag("site", "ack").ShouldBeGreaterThanOrEqualTo(1);
|
||||
}, duration: Timeout);
|
||||
}
|
||||
|
||||
// ---------------- alarm-emit gate ----------------
|
||||
|
||||
/// <summary>Role unknown + count 2 ⇒ ForwardNativeAlarm still writes the (ungated) OPC UA condition update
|
||||
/// but publishes NO cluster-wide alerts transition.</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_multi_driver_suppresses_alerts_emit_but_updates_condition()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var dep = SeedAlarmTagDeployment(db);
|
||||
var alerts = CreateTestProbe();
|
||||
SubscribeToAlerts(alerts);
|
||||
var (actor, publish) = SpawnAlarmHost(db, dep, driverMemberCount: 2);
|
||||
|
||||
actor.Tell(RaiseAlarm());
|
||||
|
||||
// The ungated OPC UA condition write still arrives.
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout).AlarmNodeId.ShouldBe("eq-1/temp_hi");
|
||||
// The cluster-wide alerts publish is suppressed (a real Primary peer publishes the single copy).
|
||||
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Role unknown + count 1 ⇒ ForwardNativeAlarm publishes the alerts transition (single-node
|
||||
/// posture preserved).</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_single_driver_publishes_alerts_emit()
|
||||
{
|
||||
var db = NewInMemoryDbFactory();
|
||||
var dep = SeedAlarmTagDeployment(db);
|
||||
var alerts = CreateTestProbe();
|
||||
SubscribeToAlerts(alerts);
|
||||
var (actor, publish) = SpawnAlarmHost(db, dep, driverMemberCount: 1);
|
||||
|
||||
actor.Tell(RaiseAlarm());
|
||||
|
||||
publish.ExpectMsg<OpcUaPublishActor.AlarmStateUpdate>(Timeout);
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(Timeout).AlarmId.ShouldBe("eq-1/temp_hi");
|
||||
}
|
||||
|
||||
// ---------------- helpers ----------------
|
||||
|
||||
private static DriverInstanceActor.AttributeAlarmPublished RaiseAlarm() =>
|
||||
new("drv-1", new AlarmEventArgs(
|
||||
new StubAlarmHandle(),
|
||||
SourceNodeId: "Temp",
|
||||
ConditionId: "Temp.HiHi",
|
||||
AlarmType: "OffNormalAlarm",
|
||||
Message: "temperature high",
|
||||
Severity: AlarmSeverity.High,
|
||||
SourceTimestampUtc: Ts,
|
||||
Kind: AlarmTransitionKind.Raise));
|
||||
|
||||
private static void TellRole(IActorRef host, RedundancyRole role) =>
|
||||
host.Tell(new RedundancyStateChanged(
|
||||
new[]
|
||||
{
|
||||
new NodeRedundancyState(TestNode, role,
|
||||
IsClusterLeader: role == RedundancyRole.Primary,
|
||||
IsRoleLeaderForDriver: role == RedundancyRole.Primary,
|
||||
AsOfUtc: DateTime.UtcNow),
|
||||
},
|
||||
CorrelationId.NewId()));
|
||||
|
||||
private IActorRef SpawnWriteHost(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId dep, IDriverFactory factory, int driverMemberCount)
|
||||
{
|
||||
var coordinator = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
driverFactory: factory,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
driverMemberCountProvider: () => driverMemberCount));
|
||||
|
||||
actor.Tell(new DispatchDeployment(dep, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
return actor;
|
||||
}
|
||||
|
||||
private (IActorRef Actor, TestProbe Publish) SpawnAlarmHost(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId dep, int driverMemberCount)
|
||||
{
|
||||
var coordinator = CreateTestProbe();
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var vtHost = CreateTestProbe();
|
||||
|
||||
var actor = Sys.ActorOf(DriverHostActor.Props(
|
||||
db, TestNode, coordinator.Ref,
|
||||
localRoles: new HashSet<string> { "driver" },
|
||||
dependencyMux: mux.Ref,
|
||||
opcUaPublishActor: publish.Ref,
|
||||
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
|
||||
virtualTagHostOverride: vtHost.Ref,
|
||||
driverMemberCountProvider: () => driverMemberCount));
|
||||
|
||||
actor.Tell(new DispatchDeployment(dep, RevA, CorrelationId.NewId()));
|
||||
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
|
||||
publish.ExpectMsg<OpcUaPublishActor.RebuildAddressSpace>(Timeout);
|
||||
return (actor, publish);
|
||||
}
|
||||
|
||||
private void SubscribeToAlerts(TestProbe probe)
|
||||
{
|
||||
DistributedPubSub.Get(Sys).Mediator.Tell(
|
||||
new Subscribe(ScriptedAlarmHostActor.AlertsTopic, probe.Ref), probe.Ref);
|
||||
probe.ExpectMsg<SubscribeAck>(Timeout);
|
||||
}
|
||||
|
||||
private static DeploymentId SeedWriteTagDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db)
|
||||
{
|
||||
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
Namespaces = new[] { new { NamespaceId = "ns-eq", Kind = 0 } },
|
||||
DriverInstances = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
DriverInstanceRowId = Guid.NewGuid(),
|
||||
DriverInstanceId = "drv-1",
|
||||
Name = "drv-1",
|
||||
DriverType = "Modbus",
|
||||
Enabled = true,
|
||||
DriverConfig = "{}",
|
||||
NamespaceId = "ns-eq",
|
||||
},
|
||||
},
|
||||
Tags = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
TagId = "tag-0",
|
||||
EquipmentId = "eq-1",
|
||||
DriverInstanceId = "drv-1",
|
||||
Name = "speed",
|
||||
FolderPath = (string?)null,
|
||||
DataType = "Double",
|
||||
TagConfig = JsonSerializer.Serialize(new { FullName = "40001" }),
|
||||
},
|
||||
},
|
||||
});
|
||||
return SealArtifact(db, artifact);
|
||||
}
|
||||
|
||||
private static DeploymentId SeedAlarmTagDeployment(IDbContextFactory<OtOpcUaConfigDbContext> db)
|
||||
{
|
||||
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
Namespaces = new[] { new { NamespaceId = "ns-eq", Kind = 0 } },
|
||||
DriverInstances = new[] { new { DriverInstanceId = "drv-1", NamespaceId = "ns-eq" } },
|
||||
Tags = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
TagId = "tag-0",
|
||||
EquipmentId = "eq-1",
|
||||
DriverInstanceId = "drv-1",
|
||||
Name = "temp_hi",
|
||||
FolderPath = (string?)null,
|
||||
DataType = "Boolean",
|
||||
TagConfig = JsonSerializer.Serialize(new
|
||||
{
|
||||
FullName = "Temp.HiHi",
|
||||
alarm = new { alarmType = "OffNormalAlarm", severity = 700 },
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
return SealArtifact(db, artifact);
|
||||
}
|
||||
|
||||
private static DeploymentId SealArtifact(IDbContextFactory<OtOpcUaConfigDbContext> db, byte[] artifact)
|
||||
{
|
||||
var id = DeploymentId.NewId();
|
||||
using var ctx = db.CreateDbContext();
|
||||
ctx.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = id.Value,
|
||||
RevisionHash = RevA.Value,
|
||||
Status = DeploymentStatus.Sealed,
|
||||
CreatedBy = "test",
|
||||
SealedAtUtc = DateTime.UtcNow,
|
||||
ArtifactBlob = artifact,
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
return id;
|
||||
}
|
||||
|
||||
private sealed class StubAlarmHandle : IAlarmSubscriptionHandle
|
||||
{
|
||||
public string DiagnosticId => "stub-alarm-sub";
|
||||
}
|
||||
|
||||
private sealed class RecordingDriverFactory : IDriverFactory
|
||||
{
|
||||
private readonly string _supportedType;
|
||||
private readonly RecordingDriver _driver = new();
|
||||
public RecordingDriverFactory(string supportedType) { _supportedType = supportedType; }
|
||||
public IReadOnlyList<WriteRequest> Writes => _driver.Writes;
|
||||
|
||||
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
|
||||
{
|
||||
if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
|
||||
_driver.Bind(driverInstanceId, driverType);
|
||||
return _driver;
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
|
||||
}
|
||||
|
||||
private sealed class RecordingDriver : IDriver, IWritable
|
||||
{
|
||||
private readonly ConcurrentQueue<WriteRequest> _writes = new();
|
||||
public string DriverInstanceId { get; private set; } = string.Empty;
|
||||
public string DriverType { get; private set; } = string.Empty;
|
||||
public IReadOnlyList<WriteRequest> Writes => _writes.ToArray();
|
||||
public void Bind(string id, string type) { DriverInstanceId = id; DriverType = type; }
|
||||
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);
|
||||
public long GetMemoryFootprint() => 0;
|
||||
public Task FlushOptionalCachesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public Task<IReadOnlyList<WriteResult>> WriteAsync(
|
||||
IReadOnlyList<WriteRequest> writes, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var w in writes) _writes.Enqueue(w);
|
||||
return Task.FromResult<IReadOnlyList<WriteResult>>(writes.Select(_ => new WriteResult(0u)).ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class MeterRecorder : IDisposable
|
||||
{
|
||||
private readonly string _name;
|
||||
private readonly MeterListener _listener;
|
||||
private long _total;
|
||||
private readonly List<KeyValuePair<string, object?>[]> _tagSets = new();
|
||||
private readonly object _gate = new();
|
||||
|
||||
public MeterRecorder(string instrumentName)
|
||||
{
|
||||
_name = instrumentName;
|
||||
_listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == _name)
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
|
||||
{
|
||||
lock (_gate) { _total += value; _tagSets.Add(tags.ToArray()); }
|
||||
});
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
public long Total { get { lock (_gate) return _total; } }
|
||||
|
||||
public int WithTag(string key, string value)
|
||||
{
|
||||
lock (_gate) return _tagSets.Count(set => set.Any(t => t.Key == key && Equals(t.Value, value)));
|
||||
}
|
||||
|
||||
public void Dispose() => _listener.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
|
||||
|
||||
/// <summary>
|
||||
/// archreview 03/S4 — the single Primary data-plane gate decision. A KNOWN role wins outright; an
|
||||
/// UNKNOWN role (no snapshot yet, or the snapshot never mentioned this node) is resolved by cluster
|
||||
/// membership: a single-driver cluster stays default-ALLOW (boot-window / single-node posture), a
|
||||
/// multi-driver cluster is default-DENY (a real Primary peer exists — don't touch the shared field
|
||||
/// device until the snapshot proves this node is Primary).
|
||||
/// </summary>
|
||||
public sealed class PrimaryGatePolicyTests
|
||||
{
|
||||
[Theory]
|
||||
// Primary → always service, regardless of member count.
|
||||
[InlineData(RedundancyRole.Primary, 0, true)]
|
||||
[InlineData(RedundancyRole.Primary, 1, true)]
|
||||
[InlineData(RedundancyRole.Primary, 2, true)]
|
||||
// Secondary / Detached → never service, regardless of member count.
|
||||
[InlineData(RedundancyRole.Secondary, 0, false)]
|
||||
[InlineData(RedundancyRole.Secondary, 2, false)]
|
||||
[InlineData(RedundancyRole.Detached, 0, false)]
|
||||
[InlineData(RedundancyRole.Detached, 2, false)]
|
||||
public void Known_role_wins_regardless_of_member_count(RedundancyRole role, int members, bool expected)
|
||||
=> PrimaryGatePolicy.ShouldServiceAsPrimary(role, members).ShouldBe(expected);
|
||||
|
||||
[Theory]
|
||||
// Unknown role: allow only when no driver peer exists (count <= 1).
|
||||
[InlineData(0, true)]
|
||||
[InlineData(1, true)]
|
||||
[InlineData(2, false)]
|
||||
[InlineData(3, false)]
|
||||
public void Unknown_role_resolves_by_membership(int members, bool expected)
|
||||
=> PrimaryGatePolicy.ShouldServiceAsPrimary(null, members).ShouldBe(expected);
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
||||
using ZB.MOM.WW.OtOpcUa.OpcUaServer;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.OpcUa;
|
||||
|
||||
/// <summary>
|
||||
/// archreview 01/S-1: when the applier reports a degraded apply (a rebuild that threw, or per-node
|
||||
/// materialise failures), <see cref="OpcUaPublishActor.HandleRebuild"/> must surface it — increment the
|
||||
/// <c>otopcua.opcua.apply.failed</c> meter and log at Error — instead of the optimistic Info line. The
|
||||
/// happy path must stay Info-only (zero increments).
|
||||
/// </summary>
|
||||
public sealed class OpcUaPublishActorApplyFailureTests : RuntimeActorTestBase
|
||||
{
|
||||
/// <summary>A rebuild whose sink throws increments otopcua.opcua.apply.failed (kind=rebuild).</summary>
|
||||
[Fact]
|
||||
public void Rebuild_when_sink_throws_increments_apply_failed_meter()
|
||||
{
|
||||
using var recorder = new MeterRecorder("otopcua.opcua.apply.failed");
|
||||
var db = NewInMemoryDbFactory();
|
||||
var sink = new ThrowOnRebuildSink();
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
SeedEquipmentDeployment(db, "eq-1");
|
||||
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(sink: sink, dbFactory: db, applier: applier));
|
||||
actor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId()));
|
||||
|
||||
AwaitAssert(() =>
|
||||
{
|
||||
recorder.Total.ShouldBeGreaterThanOrEqualTo(1);
|
||||
recorder.WithTag("kind", "rebuild").ShouldBeGreaterThanOrEqualTo(1);
|
||||
}, duration: TimeSpan.FromSeconds(2));
|
||||
}
|
||||
|
||||
/// <summary>A clean rebuild does NOT increment the apply-failed meter (Info-only happy path).</summary>
|
||||
[Fact]
|
||||
public void Rebuild_happy_path_does_not_increment_apply_failed_meter()
|
||||
{
|
||||
using var recorder = new MeterRecorder("otopcua.opcua.apply.failed");
|
||||
var db = NewInMemoryDbFactory();
|
||||
var sink = new NoopSink();
|
||||
var applier = new AddressSpaceApplier(sink, NullLogger<AddressSpaceApplier>.Instance);
|
||||
SeedEquipmentDeployment(db, "eq-1");
|
||||
|
||||
var actor = Sys.ActorOf(OpcUaPublishActor.PropsForTests(sink: sink, dbFactory: db, applier: applier));
|
||||
actor.Tell(new OpcUaPublishActor.RebuildAddressSpace(CorrelationId.NewId()));
|
||||
|
||||
// Give the actor time to process, then confirm the failed meter never fired.
|
||||
Thread.Sleep(400);
|
||||
recorder.Total.ShouldBe(0);
|
||||
}
|
||||
|
||||
private void AwaitAssert(Action assertion, TimeSpan duration)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + duration;
|
||||
Exception? last = null;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
try { assertion(); return; }
|
||||
catch (Exception ex) { last = ex; Thread.Sleep(25); }
|
||||
}
|
||||
if (last is not null) throw last;
|
||||
}
|
||||
|
||||
private static void SeedEquipmentDeployment(IDbContextFactory<OtOpcUaConfigDbContext> dbFactory, params string[] equipmentIds)
|
||||
{
|
||||
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
Equipment = equipmentIds.Select(id => new
|
||||
{
|
||||
EquipmentId = id,
|
||||
MachineCode = id.ToUpperInvariant(),
|
||||
UnsLineId = "line-1",
|
||||
Name = id,
|
||||
}).ToArray(),
|
||||
DriverInstances = Array.Empty<object>(),
|
||||
ScriptedAlarms = Array.Empty<object>(),
|
||||
});
|
||||
|
||||
using var ctx = dbFactory.CreateDbContext();
|
||||
ctx.Deployments.Add(new Deployment
|
||||
{
|
||||
DeploymentId = Guid.NewGuid(),
|
||||
RevisionHash = new string('a', 64),
|
||||
Status = DeploymentStatus.Sealed,
|
||||
CreatedBy = "test",
|
||||
SealedAtUtc = DateTime.UtcNow,
|
||||
ArtifactBlob = artifact,
|
||||
});
|
||||
ctx.SaveChanges();
|
||||
}
|
||||
|
||||
/// <summary>A sink whose RebuildAddressSpace throws (drives the applier's SafeRebuild catch → RebuildFailed).</summary>
|
||||
private sealed class ThrowOnRebuildSink : IOpcUaAddressSpaceSink
|
||||
{
|
||||
public void WriteValue(string nodeId, object? value, OpcUaQuality quality, DateTime sourceTimestampUtc) { }
|
||||
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc) { }
|
||||
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, bool isNative = false) { }
|
||||
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName) { }
|
||||
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, string? historianTagname = null, bool isArray = false, uint? arrayLength = null) { }
|
||||
public void RebuildAddressSpace() => throw new InvalidOperationException("simulated rebuild fault");
|
||||
public void RaiseNodesAddedModelChange(string affectedNodeId) { }
|
||||
}
|
||||
|
||||
/// <summary>A no-op sink — a clean apply (no failures).</summary>
|
||||
private sealed class NoopSink : IOpcUaAddressSpaceSink
|
||||
{
|
||||
public void WriteValue(string nodeId, object? value, OpcUaQuality quality, DateTime sourceTimestampUtc) { }
|
||||
public void WriteAlarmCondition(string alarmNodeId, AlarmConditionSnapshot state, DateTime occurredUtc) { }
|
||||
public void MaterialiseAlarmCondition(string alarmNodeId, string equipmentNodeId, string displayName, string alarmType, int severity, bool isNative = false) { }
|
||||
public void EnsureFolder(string folderNodeId, string? parentNodeId, string displayName) { }
|
||||
public void EnsureVariable(string variableNodeId, string? parentFolderNodeId, string displayName, string dataType, bool writable, string? historianTagname = null, bool isArray = false, uint? arrayLength = null) { }
|
||||
public void RebuildAddressSpace() { }
|
||||
public void RaiseNodesAddedModelChange(string affectedNodeId) { }
|
||||
}
|
||||
|
||||
/// <summary>Listens to a single instrument by name on the central meter and tallies value + tags.</summary>
|
||||
private sealed class MeterRecorder : IDisposable
|
||||
{
|
||||
private readonly string _name;
|
||||
private readonly MeterListener _listener;
|
||||
private long _total;
|
||||
private readonly List<KeyValuePair<string, object?>[]> _tagSets = new();
|
||||
private readonly object _gate = new();
|
||||
|
||||
public MeterRecorder(string instrumentName)
|
||||
{
|
||||
_name = instrumentName;
|
||||
_listener = new MeterListener
|
||||
{
|
||||
InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == OtOpcUaTelemetry.MeterName && instrument.Name == _name)
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
_listener.SetMeasurementEventCallback<long>((_, value, tags, _) =>
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_total += value;
|
||||
_tagSets.Add(tags.ToArray());
|
||||
}
|
||||
});
|
||||
_listener.Start();
|
||||
}
|
||||
|
||||
public long Total { get { lock (_gate) return _total; } }
|
||||
|
||||
public int WithTag(string key, string value)
|
||||
{
|
||||
lock (_gate) return _tagSets.Count(set => set.Any(t => t.Key == key && Equals(t.Value, value)));
|
||||
}
|
||||
|
||||
public void Dispose() => _listener.Dispose();
|
||||
}
|
||||
}
|
||||
+52
@@ -519,6 +519,58 @@ public sealed class ScriptedAlarmHostActorTests : RuntimeActorTestBase
|
||||
evt.TransitionKind.ShouldBe("Activated");
|
||||
}
|
||||
|
||||
/// <summary>Boot-window multi-driver suppression (archreview 03/S4): role unknown + a real driver peer
|
||||
/// (member count 2) ⇒ the alerts publish is DENIED (default-deny closes the dual-primary window that would
|
||||
/// historize duplicate AVEVA rows), while the OPC UA node write stays ungated.</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_multi_driver_suppresses_alerts_but_still_writes_opcua()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var alerts = CreateTestProbe();
|
||||
SubscribeToAlerts(alerts);
|
||||
|
||||
var (host, _) = SpawnWithMemberCount(publish, mux, LocalNode, driverMemberCount: 2);
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(severity: 800) }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
|
||||
// No snapshot sent ⇒ role unknown; count 2 ⇒ default-DENY.
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
|
||||
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(m => m.State.Active, Timeout);
|
||||
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
||||
}
|
||||
|
||||
/// <summary>Boot-window single-driver emit (archreview 03/S4): role unknown + no driver peer (member
|
||||
/// count 1) ⇒ the alerts publish proceeds (single-node posture preserved).</summary>
|
||||
[Fact]
|
||||
public void Unknown_role_single_driver_publishes_alerts()
|
||||
{
|
||||
var publish = CreateTestProbe();
|
||||
var mux = CreateTestProbe();
|
||||
var alerts = CreateTestProbe();
|
||||
SubscribeToAlerts(alerts);
|
||||
|
||||
var (host, _) = SpawnWithMemberCount(publish, mux, LocalNode, driverMemberCount: 1);
|
||||
host.Tell(new ScriptedAlarmHostActor.ApplyScriptedAlarms(new[] { Plan(severity: 800) }));
|
||||
mux.ExpectMsg<DependencyMuxActor.RegisterInterest>(Timeout);
|
||||
|
||||
host.Tell(new VirtualTagActor.DependencyValueChanged("M.T", 99, DateTime.UtcNow));
|
||||
|
||||
publish.FishForMessage<OpcUaPublishActor.AlarmStateUpdate>(m => m.State.Active, Timeout);
|
||||
alerts.ExpectMsg<AlarmTransitionEvent>(Timeout).AlarmId.ShouldBe("alm-1");
|
||||
}
|
||||
|
||||
private (IActorRef Host, DependencyMuxTagUpstreamSource Upstream) SpawnWithMemberCount(
|
||||
TestProbe publish, TestProbe mux, NodeId localNode, int driverMemberCount)
|
||||
{
|
||||
var upstream = new DependencyMuxTagUpstreamSource();
|
||||
var engine = BuildEngine(upstream);
|
||||
var host = Sys.ActorOf(ScriptedAlarmHostActor.Props(
|
||||
publish.Ref, mux.Ref, upstream, engine, localNode, driverMemberCountProvider: () => driverMemberCount));
|
||||
return (host, upstream);
|
||||
}
|
||||
|
||||
/// <summary>Inbound command ungated by role (T1): the alerts-publish gate must NOT affect inbound
|
||||
/// command processing. Under a Secondary role, an AlarmCommand("Acknowledge") for an owned, active
|
||||
/// alarm still drives the engine — observed via the resulting AlarmStateUpdate(Acknowledged=true)
|
||||
|
||||
Reference in New Issue
Block a user