using System.Collections.Concurrent;
using System.Diagnostics.Metrics;
using System.Text.Json;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.TestKit;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Alerts;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Observability;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.OpcUa;
using ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
///
/// archreview 03/S4 — the boot-window primary-gate guard. On a MULTI-driver cluster (member count > 1)
/// while the role is UNKNOWN, the Primary data-plane gates (inbound write, native ack, native alerts emit)
/// default-DENY; on a SINGLE-driver cluster (count <= 1) they default-ALLOW (boot-window / single-node
/// posture). Driver member count is injected via the driverMemberCountProvider Props seam.
///
public sealed class DriverHostActorPrimaryGateTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-pgate-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly DateTime Ts = new(2026, 7, 13, 10, 0, 0, DateTimeKind.Utc);
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
/// v3 RawPath of the seeded write tag: RawFolder "Plant" / driver Name "Modbus" / Device "dev1" /
/// Tag "speed". Its UNS projection (Area "filling" / Line "line1" / Equipment "station1") is
/// .
private const string WriteRawPath = "Plant/Modbus/dev1/speed";
private const string WriteUnsNodeId = "filling/line1/station1/speed";
/// v3 RawPath of the seeded alarm tag. In v3 a native alarm materializes ONCE at the raw tag with
/// ConditionId == RawPath, so this is both the wire-ref the driver raises and the condition
/// NodeId the host routes to.
private const string AlarmRawPath = "Plant/Modbus/dev1/temp_hi";
// ---------------- write gate ----------------
/// Role unknown + a real driver peer (count 2) ⇒ RouteNodeWrite is DENIED with the
/// boot-window reason and the driver sees no write.
[Fact]
public void Unknown_role_multi_driver_denies_write_with_role_unknown_reason()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingDriverFactory("Modbus");
var dep = SeedWriteTagDeployment(db);
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 2);
var asker = CreateTestProbe();
actor.Tell(new DriverHostActor.RouteNodeWrite(WriteUnsNodeId, 123.0, AddressSpaceRealm.Uns), asker.Ref);
var result = asker.ExpectMsg(Timeout);
result.Success.ShouldBeFalse();
result.Reason.ShouldBe("not primary (role unknown)");
recorder.Writes.ShouldBeEmpty();
}
/// Role unknown + no driver peer (count 1) ⇒ the write is SERVICED (single-node / boot-window
/// posture preserved).
[Fact]
public void Unknown_role_single_driver_services_write()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingDriverFactory("Modbus");
var dep = SeedWriteTagDeployment(db);
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 1);
// The gate must ALLOW (count 1, role unknown). The driver child's async connect can lag the ApplyAck,
// so a first write may transiently report "driver not connected" — retry until connected (a real
// client retries too). The point under test is that the gate never denies (never "not primary").
AwaitAssert(() =>
{
var asker = CreateTestProbe();
actor.Tell(new DriverHostActor.RouteNodeWrite(WriteUnsNodeId, 123.0, AddressSpaceRealm.Uns), asker.Ref);
var res = asker.ExpectMsg(Timeout);
res.Reason.ShouldNotBe("not primary");
res.Reason.ShouldNotBe("not primary (role unknown)");
res.Success.ShouldBeTrue(res.Reason);
}, duration: TimeSpan.FromSeconds(10));
recorder.Writes.ShouldNotBeEmpty();
}
/// A Primary snapshot services even at count 2; a Secondary snapshot denies with the steady-state
/// reason (distinct from the boot-window reason).
[Fact]
public void Known_role_wins_over_member_count()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingDriverFactory("Modbus");
var dep = SeedWriteTagDeployment(db);
var actor = SpawnWriteHost(db, dep, recorder, driverMemberCount: 2);
// Primary snapshot ⇒ serviced even with a driver peer. Retry through the transient driver-connect
// window; the invariant under test is that the gate never denies a Primary.
TellRole(actor, RedundancyRole.Primary);
AwaitAssert(() =>
{
var asker1 = CreateTestProbe();
actor.Tell(new DriverHostActor.RouteNodeWrite(WriteUnsNodeId, 1.0, AddressSpaceRealm.Uns), asker1.Ref);
var res = asker1.ExpectMsg(Timeout);
res.Reason.ShouldNotBe("not primary");
res.Success.ShouldBeTrue(res.Reason);
}, duration: TimeSpan.FromSeconds(10));
// Secondary snapshot ⇒ denied with the steady-state reason.
TellRole(actor, RedundancyRole.Secondary);
var asker2 = CreateTestProbe();
actor.Tell(new DriverHostActor.RouteNodeWrite(WriteUnsNodeId, 2.0, AddressSpaceRealm.Uns), asker2.Ref);
var denied = asker2.ExpectMsg(Timeout);
denied.Success.ShouldBeFalse();
denied.Reason.ShouldBe("not primary");
}
/// The denial meter carries the site + reason tags.
[Fact]
public void Denied_write_increments_primary_gate_denied_meter_with_tags()
{
using var recorder = new MeterRecorder("otopcua.redundancy.primary_gate_denied");
var db = NewInMemoryDbFactory();
var factory = new RecordingDriverFactory("Modbus");
var dep = SeedWriteTagDeployment(db);
var actor = SpawnWriteHost(db, dep, factory, driverMemberCount: 2);
var asker = CreateTestProbe();
actor.Tell(new DriverHostActor.RouteNodeWrite(WriteUnsNodeId, 9.0, AddressSpaceRealm.Uns), asker.Ref);
asker.ExpectMsg(Timeout).Success.ShouldBeFalse();
AwaitAssert(() =>
{
recorder.Total.ShouldBeGreaterThanOrEqualTo(1);
recorder.WithTag("site", "write").ShouldBeGreaterThanOrEqualTo(1);
recorder.WithTag("reason", "role-unknown").ShouldBeGreaterThanOrEqualTo(1);
}, duration: Timeout);
}
// ---------------- ack gate ----------------
/// Role unknown + count 2 ⇒ a native-alarm ack is dropped at Warning and the denial meter
/// increments with site=ack.
[Fact]
public void Unknown_role_multi_driver_drops_native_ack_with_warning_and_meter()
{
using var meter = new MeterRecorder("otopcua.redundancy.primary_gate_denied");
var db = NewInMemoryDbFactory();
var dep = SeedAlarmTagDeployment(db);
var (actor, _) = SpawnAlarmHost(db, dep, driverMemberCount: 2);
EventFilter.Warning(contains: "role unknown").ExpectOne(() =>
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, Comment: null, OperatorUser: "op")));
AwaitAssert(() =>
{
meter.WithTag("site", "ack").ShouldBeGreaterThanOrEqualTo(1);
}, duration: Timeout);
}
// ---------------- alarm-emit gate ----------------
/// Role unknown + count 2 ⇒ ForwardNativeAlarm still writes the (ungated) OPC UA condition update
/// but publishes NO cluster-wide alerts transition.
[Fact]
public void Unknown_role_multi_driver_suppresses_alerts_emit_but_updates_condition()
{
var db = NewInMemoryDbFactory();
var dep = SeedAlarmTagDeployment(db);
var alerts = CreateTestProbe();
SubscribeToAlerts(alerts);
var (actor, publish) = SpawnAlarmHost(db, dep, driverMemberCount: 2);
actor.Tell(RaiseAlarm());
// The ungated OPC UA condition write still arrives.
publish.ExpectMsg(Timeout).AlarmNodeId.ShouldBe(AlarmRawPath);
// The cluster-wide alerts publish is suppressed (a real Primary peer publishes the single copy).
alerts.ExpectNoMsg(TimeSpan.FromMilliseconds(500));
}
/// Role unknown + count 1 ⇒ ForwardNativeAlarm publishes the alerts transition (single-node
/// posture preserved).
[Fact]
public void Unknown_role_single_driver_publishes_alerts_emit()
{
var db = NewInMemoryDbFactory();
var dep = SeedAlarmTagDeployment(db);
var alerts = CreateTestProbe();
SubscribeToAlerts(alerts);
var (actor, publish) = SpawnAlarmHost(db, dep, driverMemberCount: 1);
actor.Tell(RaiseAlarm());
publish.ExpectMsg(Timeout);
alerts.ExpectMsg(Timeout).AlarmId.ShouldBe(AlarmRawPath);
}
// ---------------- helpers ----------------
private static DriverInstanceActor.AttributeAlarmPublished RaiseAlarm() =>
new("drv-1", new AlarmEventArgs(
new StubAlarmHandle(),
SourceNodeId: "Temp", // bare owning object — deliberately NOT the lookup key
ConditionId: AlarmRawPath, // v3: the condition is keyed by the raw tag's RawPath
AlarmType: "OffNormalAlarm",
Message: "temperature high",
Severity: AlarmSeverity.High,
SourceTimestampUtc: Ts,
Kind: AlarmTransitionKind.Raise));
private static void TellRole(IActorRef host, RedundancyRole role) =>
host.Tell(new RedundancyStateChanged(
new[]
{
new NodeRedundancyState(TestNode, role,
IsClusterLeader: role == RedundancyRole.Primary,
IsDriverPrimary: role == RedundancyRole.Primary,
AsOfUtc: DateTime.UtcNow),
},
CorrelationId.NewId()));
private IActorRef SpawnWriteHost(
IDbContextFactory db, DeploymentId dep, IDriverFactory factory, int driverMemberCount)
{
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet { "driver" },
driverMemberCountProvider: () => driverMemberCount));
actor.Tell(new DispatchDeployment(dep, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
return actor;
}
private (IActorRef Actor, TestProbe Publish) SpawnAlarmHost(
IDbContextFactory db, DeploymentId dep, int driverMemberCount)
{
var coordinator = CreateTestProbe();
var publish = CreateTestProbe();
var mux = CreateTestProbe();
var vtHost = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
localRoles: new HashSet { "driver" },
dependencyMux: mux.Ref,
opcUaPublishActor: publish.Ref,
virtualTagEvaluator: NullVirtualTagEvaluator.Instance,
virtualTagHostOverride: vtHost.Ref,
driverMemberCountProvider: () => driverMemberCount));
actor.Tell(new DispatchDeployment(dep, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
publish.ExpectMsg(Timeout);
return (actor, publish);
}
private void SubscribeToAlerts(TestProbe probe)
{
DistributedPubSub.Get(Sys).Mediator.Tell(
new Subscribe(ScriptedAlarmHostActor.AlertsTopic, probe.Ref), probe.Ref);
probe.ExpectMsg(Timeout);
}
///
/// Seeds the v3 raw-tag chain a write must traverse: RawFolder "Plant" → DriverInstance "drv-1"
/// (Modbus, Enabled so a real child spawns and can service the write) → Device "dev1" → Tag
/// "speed" (, ReadWrite), projected into equipment
/// filling/line1/station1 by a UnsTagReference so resolves.
/// The pre-v3 shape this replaced seeded Namespaces + equipment-bound Tags, which
/// v3 parses to an EMPTY tag set — the write had nothing to route to, which is why the
/// write-is-serviced cases were skipped rather than failing.
///
private static DeploymentId SeedWriteTagDeployment(IDbContextFactory db)
{
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new { DriverInstanceId = "drv-1", RawFolderId = "rf-plant", Name = "Modbus", DriverType = "Modbus", DriverConfig = "{}", ClusterId = "c1", Enabled = true },
},
Devices = new[] { new { DeviceId = "dev-1", DriverInstanceId = "drv-1", Name = "dev1", DeviceConfig = "{}" } },
TagGroups = Array.Empty