Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/Drivers/DriverHostActorNativeAlarmAckRoutingTests.cs
T
Joseph Doherty 50b55ee4b9 fix(redundancy): elect the Primary by cluster age, not by address
RedundancyStateActor derived the driver Primary from ClusterState.RoleLeader("driver").
Akka offers two different notions of a "first" member and they are not the same
one: role leader is the lowest-ADDRESSED Up member (host, then port), while
ClusterSingletonManager places singletons on the OLDEST — lowest up-number.

They agree on a freshly-formed cluster, which is why every existing test passed.
They diverge after any restart: the restarted node re-joins as the youngest while
keeping its address, so if it holds the lower address it becomes role leader while
the singletons stay put. The snapshot would then name a Primary that is not
hosting the work, and every Primary-gated surface follows it — inbound device
writes, native-alarm acks, the fleet-wide alerts emit, and the alarm-history
drain would all enable on the wrong node while the node actually running the
singletons stayed gated off.

BuildSnapshot now selects the oldest Up member carrying the driver role, matching
singleton placement. Leaving members are excluded: a node handing its singletons
over must not be named Primary.

NodeRedundancyState.IsRoleLeaderForDriver is renamed IsDriverPrimary, and
NodeHealthInputs.IsDriverRoleLeader likewise. Keeping the old names would have
left the wire contract asserting a derivation the code no longer uses — the same
drift that made this defect invisible.

Proven by a real two-node cluster rather than a mock. RedundancyPrimaryElectionTests
binds the first-joining node to the HIGHER port, so oldest and lowest-address name
different nodes, and includes a fixture assertion that the divergence actually
occurred — without it the real assertion could pass for the wrong reason. Positive
control: restoring the RoleLeader derivation turns exactly the two election tests
red while the fixture check stays green.

Runtime.Tests 440 passed, ControlPlane.Tests 82, Cluster.Tests 36.

Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-21 18:39:07 -04:00

278 lines
13 KiB
C#

using System.Collections.Concurrent;
using System.Text.Json;
using Akka.Actor;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Commons.Engines;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Fleet;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Redundancy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Drivers;
using ZB.MOM.WW.OtOpcUa.Runtime.Tests.Harness;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Drivers;
/// <summary>
/// v3 Batch 4 (B4-WP4) — the inbound native-condition <b>acknowledge</b> routing wired into
/// <see cref="DriverHostActor"/> re-expressed for the v3 raw-condition model. An OPC UA client Acknowledges
/// a NATIVE condition (materialised at the raw tag, ConditionId = its RawPath); the node manager invokes
/// <c>NativeAlarmAckRouter</c> and the host receives a <see cref="DriverHostActor.RouteNativeAlarmAck"/>.
/// The host resolves the condition NodeId (== the RawPath) → owning <c>(DriverInstanceId, RawPath)</c> via
/// the <c>_driverRefByAlarmNodeId</c> inverse map (built alongside the alarm forward map in
/// <c>PushDesiredSubscriptions</c> from the alarm-bearing <c>composition.RawTags</c>), applies the SAME
/// primary gate the inbound write path uses, and routes to the owning driver child's
/// <see cref="IAlarmSource.AcknowledgeAsync"/> carrying the principal.
///
/// <para>
/// Mirrors <c>DriverHostActorLiveValueTests</c>: a real apply through the harness spawns a real
/// (non-stubbed) <see cref="DriverInstanceActor"/> child backed by a recording <see cref="IAlarmSource"/>
/// driver (DriverType "GalaxyMxGateway", Enabled), so the inverse map is populated authentically and the
/// forwarded acknowledge request can be observed. The seeded raw tag carries an <c>alarm</c> object so it
/// materialises as a Part 9 condition at its RawPath, not a value variable.
/// </para>
/// </summary>
public sealed class DriverHostActorNativeAlarmAckRoutingTests : RuntimeActorTestBase
{
private static readonly NodeId TestNode = NodeId.Parse("driver-ack-test");
private static readonly RevisionHash RevA = RevisionHash.Parse(new string('a', 64));
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(5);
// The v3 RawPath for the seeded alarm tag: RawFolder "Plant" / DriverName "gw" / Device "dev1" / Tag.
private const string AlarmRawPath = "Plant/gw/dev1/temp_hi";
/// <summary>On the PRIMARY (role unknown ⇒ Primary), a RouteNativeAlarmAck for a mapped raw condition NodeId
/// (== the RawPath) forwards exactly one <see cref="AlarmAcknowledgeRequest"/> to the owning driver's
/// <see cref="IAlarmSource.AcknowledgeAsync"/>, correlated on the RawPath, with the operator principal + the
/// comment.</summary>
[Fact]
public void RouteNativeAlarmAck_routes_to_driver_AcknowledgeAsync_with_principal()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
var actor = SpawnHostAndApply(db, deploymentId, recorder);
// Local role unknown ⇒ treated as Primary ⇒ ack allowed (default-allow semantics).
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
// The driver received exactly one acknowledge, correlated on its wire-ref RawPath, with principal.
AwaitAssert(() =>
{
recorder.Acks.Count.ShouldBe(1);
recorder.Acks[0].ConditionId.ShouldBe(AlarmRawPath);
recorder.Acks[0].SourceNodeId.ShouldBe(AlarmRawPath);
recorder.Acks[0].Comment.ShouldBe("cmt");
recorder.Acks[0].OperatorUser.ShouldBe("alice");
}, duration: Timeout);
}
/// <summary>An ack for an unmapped condition NodeId is dropped: no throw, and the driver's
/// <see cref="IAlarmSource.AcknowledgeAsync"/> is never called.</summary>
[Fact]
public void RouteNativeAlarmAck_unknown_node_is_dropped()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
var actor = SpawnHostAndApply(db, deploymentId, recorder);
actor.Tell(new DriverHostActor.RouteNativeAlarmAck("Plant/gw/dev1/does-not-exist", "cmt", "alice"));
// Give the (fire-and-forget) handler time to run; the unmapped node must produce no ack.
AwaitAssert(() => recorder.Acks.ShouldBeEmpty(), duration: TimeSpan.FromMilliseconds(800));
}
/// <summary>On a SECONDARY node the ack is gated off (same primary gate as the inbound write path): the
/// driver's <see cref="IAlarmSource.AcknowledgeAsync"/> is NOT called.</summary>
[Fact]
public void RouteNativeAlarmAck_on_non_primary_is_dropped()
{
var db = NewInMemoryDbFactory();
var recorder = new RecordingAlarmDriverFactory("GalaxyMxGateway");
var deploymentId = SeedV3AlarmDeployment(db, RevA, Driver: "drv-1", Tag: "temp_hi");
var actor = SpawnHostAndApply(db, deploymentId, recorder);
// Force this node Secondary so the primary gate rejects the ack.
actor.Tell(new RedundancyStateChanged(
new[]
{
new NodeRedundancyState(TestNode, RedundancyRole.Secondary,
IsClusterLeader: false, IsDriverPrimary: false, AsOfUtc: DateTime.UtcNow),
},
CorrelationId.NewId()));
actor.Tell(new DriverHostActor.RouteNativeAlarmAck(AlarmRawPath, "cmt", "alice"));
// No ack reached the driver — the gate short-circuited before the inverse-map lookup.
AwaitAssert(() => recorder.Acks.ShouldBeEmpty(), duration: TimeSpan.FromMilliseconds(800));
}
/// <summary>Spawns the host with the recording alarm-driver factory, dispatches the deployment, and waits
/// for the Applied ACK so the apply (and thus the inverse-map build in PushDesiredSubscriptions) has
/// completed before the test routes an ack.</summary>
private IActorRef SpawnHostAndApply(
IDbContextFactory<OtOpcUaConfigDbContext> db, DeploymentId deploymentId, IDriverFactory factory)
{
var coordinator = CreateTestProbe();
var actor = Sys.ActorOf(DriverHostActor.Props(
db, TestNode, coordinator.Ref,
driverFactory: factory,
localRoles: new HashSet<string> { "driver" }));
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(Timeout).Outcome.ShouldBe(ApplyAckOutcome.Applied);
return actor;
}
/// <summary>
/// Seeds a Sealed deployment whose artifact carries the v3 raw-tag chain (RawFolder "Plant" →
/// DriverInstance(RawFolderId, Name "gw", DriverType "GalaxyMxGateway", Enabled) → Device "dev1" → Tag)
/// with the tag's <c>TagConfig</c> carrying an <c>alarm</c> object so the composer projects a non-null
/// <c>EquipmentTagAlarmInfo</c> — making the raw tag a Part 9 condition at its RawPath. The non-Windows
/// <c>DriverType</c> + Enabled flag spawn a REAL <see cref="DriverInstanceActor"/> child.
/// </summary>
private static DeploymentId SeedV3AlarmDeployment(
IDbContextFactory<OtOpcUaConfigDbContext> db, RevisionHash rev, string Driver, string Tag)
{
var artifact = JsonSerializer.SerializeToUtf8Bytes(new
{
RawFolders = new[] { new { RawFolderId = "rf-plant", ParentRawFolderId = (string?)null, Name = "Plant", ClusterId = "c1" } },
DriverInstances = new[]
{
new
{
DriverInstanceRowId = Guid.NewGuid(),
DriverInstanceId = Driver,
RawFolderId = "rf-plant",
Name = "gw",
DriverType = "GalaxyMxGateway", // not Windows-only ⇒ a real child is spawned (not stubbed)
Enabled = true,
DriverConfig = "{}",
ClusterId = "c1",
},
},
Devices = new[]
{
new { DeviceId = $"{Driver}:dev1", DriverInstanceId = Driver, Name = "dev1", DeviceConfig = "{}" },
},
TagGroups = Array.Empty<object>(),
Tags = new[]
{
new
{
TagId = "tag-0",
DeviceId = $"{Driver}:dev1",
TagGroupId = (string?)null,
Name = Tag,
DataType = "Boolean",
AccessLevel = 0, // TagAccessLevel.Read
TagConfig = JsonSerializer.Serialize(new { alarm = new { alarmType = "OffNormalAlarm", severity = 700 } }),
},
},
});
var id = DeploymentId.NewId();
using var ctx = db.CreateDbContext();
ctx.Deployments.Add(new Deployment
{
DeploymentId = id.Value,
RevisionHash = rev.Value,
Status = DeploymentStatus.Sealed,
CreatedBy = "test",
SealedAtUtc = DateTime.UtcNow,
ArtifactBlob = artifact,
});
ctx.SaveChanges();
return id;
}
/// <summary>Factory producing a single <see cref="RecordingAlarmDriver"/> for the supported type, whose
/// recorded acknowledge list is exposed for assertions.</summary>
private sealed class RecordingAlarmDriverFactory : IDriverFactory
{
private readonly string _supportedType;
private readonly RecordingAlarmDriver _driver = new();
public RecordingAlarmDriverFactory(string supportedType) { _supportedType = supportedType; }
/// <summary>The acknowledge requests the spawned driver received (thread-safe — AcknowledgeAsync runs
/// off the actor thread).</summary>
public IReadOnlyList<AlarmAcknowledgeRequest> Acks => _driver.Acks;
/// <inheritdoc />
public IDriver? TryCreate(string driverType, string driverInstanceId, string driverConfigJson)
{
if (!string.Equals(driverType, _supportedType, StringComparison.Ordinal)) return null;
_driver.Bind(driverInstanceId, driverType);
return _driver;
}
/// <inheritdoc />
public IReadOnlyCollection<string> SupportedTypes => new[] { _supportedType };
}
/// <summary>An <see cref="IDriver"/> + <see cref="IAlarmSource"/> that records every acknowledge.</summary>
private sealed class RecordingAlarmDriver : IDriver, IAlarmSource
{
private readonly ConcurrentQueue<AlarmAcknowledgeRequest> _acks = new();
/// <inheritdoc />
public string DriverInstanceId { get; private set; } = string.Empty;
/// <inheritdoc />
public string DriverType { get; private set; } = string.Empty;
/// <summary>The acknowledge requests received so far.</summary>
public IReadOnlyList<AlarmAcknowledgeRequest> Acks => _acks.ToArray();
/// <summary>Sets the identity once the factory is asked to create it.</summary>
public void Bind(string id, string type) { DriverInstanceId = id; DriverType = type; }
/// <inheritdoc />
public Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task ShutdownAsync(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public DriverHealth GetHealth() => new(DriverState.Healthy, DateTime.UtcNow, LastError: null);
/// <inheritdoc />
public long GetMemoryFootprint() => 0;
/// <inheritdoc />
public Task FlushOptionalCachesAsync(CancellationToken cancellationToken) => Task.CompletedTask;
/// <inheritdoc />
public Task<IAlarmSubscriptionHandle> SubscribeAlarmsAsync(
IReadOnlyList<string> sourceNodeIds, CancellationToken cancellationToken) =>
Task.FromResult<IAlarmSubscriptionHandle>(new StubAlarmHandle());
/// <inheritdoc />
public Task UnsubscribeAlarmsAsync(IAlarmSubscriptionHandle handle, CancellationToken cancellationToken) =>
Task.CompletedTask;
/// <inheritdoc />
public Task AcknowledgeAsync(
IReadOnlyList<AlarmAcknowledgeRequest> acknowledgements, CancellationToken cancellationToken)
{
foreach (var a in acknowledgements) _acks.Enqueue(a);
return Task.CompletedTask;
}
/// <inheritdoc />
public event EventHandler<AlarmEventArgs>? OnAlarmEvent
{
add { }
remove { }
}
private sealed class StubAlarmHandle : IAlarmSubscriptionHandle
{
public string DiagnosticId => "stub-alarm-sub";
}
}
}