Files
lmxopcua/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ServiceCollectionExtensionsTests.cs
T
Joseph Doherty 667639e6ab fix(mesh): per-endpoint deadline + review fixes — failover survives a hanging central
CRITICAL. The ack deadline was one absolute DateTime computed once for the whole
endpoint walk. Against the characteristic WAN failure — a central that HANGS
(SYN black-hole, frozen host) rather than refusing — endpoint 1 burned the whole
AckTimeoutSeconds and endpoint 2 was then dialled with a deadline already in the
past: instant DeadlineExceeded, no packet leaving the node, the surviving central
never really tried. Failover looked like it ran and did not. It also contradicted
MeshTransportOptions' own documented "applied per endpoint, not per ack". The
deadline is now computed fresh inside the loop; headers stay hoisted (read-only,
sequential walk).

Two test gaps that let that survive are closed: nothing pinned the ackRouter gate
(narrowing it back to ClusterClient-only compiled and passed every test while
dead-ending every Grpc ack to DPS), and the deadline fact's lower bound accepted
any positive timeout at all. Both new pins are mutation-verified.

Docs corrected where they overstated the case: duplicate acks CANNOT mis-seal a
deployment — ConfigPublishCoordinator.HandleAck keys _acks by NodeId behind an
expected-set guard, so a duplicate overwrites its own entry. The single-endpoint
rule is defence in depth and a bound on wasted work, not a correctness
dependency; and the walk is at-least-once on the deadline path, not at-most-once.
2026-08-06 08:42:11 -04:00

308 lines
14 KiB
C#

using Akka.Actor;
using Akka.Hosting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests;
/// <summary>
/// Verifies <c>WithOtOpcUaRuntimeActors</c> spawns <c>DriverHostActor</c> + <c>DbHealthProbeActor</c>
/// on the host's <c>ActorSystem</c> and registers both under their marker keys. This is the
/// driver-role mirror of the admin-role <c>WithOtOpcUaControlPlaneSingletons</c> bootstrap.
/// </summary>
public sealed class ServiceCollectionExtensionsTests
{
/// <summary>Verifies that WithOtOpcUaRuntimeActors spawns driver host and DB health probe actors.</summary>
[Fact]
public async Task WithOtOpcUaRuntimeActors_spawns_driver_host_and_db_health_probe()
{
using var host = Host.CreateDefaultBuilder()
.ConfigureServices((_, services) =>
{
services.AddSingleton<IDbContextFactory<OtOpcUaConfigDbContext>>(
new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N")));
services.AddSingleton<IClusterRoleInfo>(new FakeClusterRoleInfo());
services.AddAkka("otopcua-test", (ab, _) =>
{
ab.AddHocon(@"
akka.actor.provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
akka.remote.dot-netty.tcp.hostname = ""127.0.0.1""
akka.remote.dot-netty.tcp.port = 0
akka.cluster.seed-nodes = []
akka.cluster.roles = [""driver""]
", HoconAddMode.Prepend);
ab.WithOtOpcUaRuntimeActors();
});
})
.Build();
await host.StartAsync();
try
{
var driverHost = host.Services.GetRequiredService<IRequiredActor<DriverHostActorKey>>();
var dbHealth = host.Services.GetRequiredService<IRequiredActor<DbHealthProbeActorKey>>();
var historian = host.Services.GetRequiredService<IRequiredActor<HistorianAdapterActorKey>>();
var mux = host.Services.GetRequiredService<IRequiredActor<DependencyMuxActorKey>>();
var publish = host.Services.GetRequiredService<IRequiredActor<OpcUaPublishActorKey>>();
var peerProbes = host.Services.GetRequiredService<IRequiredActor<PeerProbeSupervisorKey>>();
driverHost.ActorRef.ShouldNotBeNull();
dbHealth.ActorRef.ShouldNotBeNull();
historian.ActorRef.ShouldNotBeNull();
mux.ActorRef.ShouldNotBeNull();
publish.ActorRef.ShouldNotBeNull();
peerProbes.ActorRef.ShouldNotBeNull();
driverHost.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DriverHostActorName);
dbHealth.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DbHealthProbeActorName);
historian.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.HistorianAdapterActorName);
mux.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.DependencyMuxActorName);
publish.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.OpcUaPublishActorName);
peerProbes.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.PeerProbeSupervisorName);
}
finally
{
await host.StopAsync();
}
}
/// <summary>
/// When <c>ContinuousHistorization</c> is not enabled (no options registered), the recorder
/// actor is NOT spawned — its key does not resolve in the registry.
/// </summary>
[Fact]
public async Task Recorder_not_spawned_when_continuous_historization_disabled()
{
using var host = BuildRuntimeActorHost(extra: null);
await host.StartAsync();
try
{
var registry = host.Services.GetRequiredService<ActorRegistry>();
registry.TryGet<ContinuousHistorizationRecorderKey>(out _).ShouldBeFalse();
}
finally
{
await host.StopAsync();
}
}
/// <summary>
/// When <c>ContinuousHistorization</c> is enabled and a value-writer + outbox are registered,
/// the recorder actor IS spawned and its key resolves under the expected actor name.
/// </summary>
[Fact]
public async Task Recorder_spawned_when_enabled_with_writer_and_outbox()
{
using var host = BuildRuntimeActorHost(extra: services =>
{
services.AddSingleton(new ContinuousHistorizationOptions { Enabled = true, OutboxPath = "x" });
services.AddSingleton<IHistorianValueWriter>(new FakeValueWriter());
services.AddSingleton<IHistorizationOutbox>(new FakeOutbox());
});
await host.StartAsync();
try
{
var recorder = host.Services.GetRequiredService<IRequiredActor<ContinuousHistorizationRecorderKey>>();
recorder.ActorRef.ShouldNotBeNull();
recorder.ActorRef.Path.Name.ShouldBe(ServiceCollectionExtensions.ContinuousHistorizationRecorderActorName);
}
finally
{
await host.StopAsync();
}
}
/// <summary>
/// Under <c>MeshTransport:Mode = Grpc</c> an ApplyAck produced by the driver host reaches the
/// registered <see cref="IApplyAckSender"/>, and no ClusterClient actor exists
/// (ClusterClient→gRPC migration Task 1B.2).
/// </summary>
/// <remarks>
/// This is the fact that pins the <c>ackRouter</c> gate in <c>WithOtOpcUaRuntimeActors</c>
/// (<c>useClusterClient || useGrpc ? nodeComm : null</c>). Narrowed back to ClusterClient-only
/// that gate still compiles and every unit test still passes, while every ack on a Grpc node
/// goes out on the <c>deployment-acks</c> DPS topic instead — where, in the split-mesh topology
/// Grpc exists to serve, central is not subscribed. The whole transport would be dead and
/// nothing below this level of the stack could tell.
///
/// The dispatch names a deployment that does not exist in the in-memory ConfigDb, so the apply
/// fails and the host emits a Failed ack — deterministic, synchronous, and it exercises the
/// real <c>SendAck</c> path rather than poking the router directly.
/// </remarks>
[Fact]
public async Task Under_Grpc_a_driver_host_ack_reaches_the_ack_sender_and_no_ClusterClient_exists()
{
var sender = new RecordingAckSender();
using var host = BuildRuntimeActorHost(services =>
{
services.AddSingleton<IApplyAckSender>(sender);
services.Configure<MeshTransportOptions>(o =>
{
o.Mode = MeshTransportOptions.ModeGrpc;
o.CentralGrpcEndpoints = ["http://central-1:4055"];
o.GrpcPsk = "rig-key";
});
});
await host.StartAsync();
try
{
var system = host.Services.GetRequiredService<ActorSystem>();
var driverHost = host.Services.GetRequiredService<IRequiredActor<DriverHostActorKey>>();
driverHost.ActorRef.Tell(new DispatchDeployment(
new DeploymentId(Guid.NewGuid()),
new RevisionHash("rev-that-does-not-exist"),
new CorrelationId(Guid.NewGuid())));
// The ack travelled DriverHostActor -> NodeCommunicationActor -> IApplyAckSender. Under a
// ClusterClient-only gate it would have gone to DPS and this never completes.
var deadline = DateTime.UtcNow.AddSeconds(15);
while (sender.Sent.Count == 0 && DateTime.UtcNow < deadline)
await Task.Delay(50);
sender.Sent.Count.ShouldBe(1);
// And the Akka cross-mesh association is genuinely gone, not merely unused.
await Should.ThrowAsync<ActorNotFoundException>(() => system
.ActorSelection($"/user/{ServiceCollectionExtensions.CentralClusterClientName}")
.ResolveOne(TimeSpan.FromMilliseconds(500)));
}
finally
{
await host.StopAsync();
}
}
/// <summary>An <see cref="IApplyAckSender"/> double recording what the node comm actor handed it.</summary>
private sealed class RecordingAckSender : IApplyAckSender
{
private readonly Lock _gate = new();
private readonly List<ApplyAck> _sent = [];
/// <summary>Gets a snapshot of the acks posted through this sender.</summary>
public IReadOnlyList<ApplyAck> Sent
{
get { lock (_gate) return [.. _sent]; }
}
/// <summary>Records <paramref name="ack"/>.</summary>
/// <param name="ack">The ack handed over.</param>
/// <param name="cancellationToken">Unused.</param>
/// <returns>A completed task.</returns>
public Task SendAsync(ApplyAck ack, CancellationToken cancellationToken = default)
{
lock (_gate) _sent.Add(ack);
return Task.CompletedTask;
}
}
/// <summary>Builds a driver-role host that runs <c>WithOtOpcUaRuntimeActors</c>, with optional
/// extra DI registrations applied before <c>AddAkka</c>.</summary>
private static IHost BuildRuntimeActorHost(Action<IServiceCollection>? extra)
=> Host.CreateDefaultBuilder()
.ConfigureServices((_, services) =>
{
services.AddSingleton<IDbContextFactory<OtOpcUaConfigDbContext>>(
new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N")));
services.AddSingleton<IClusterRoleInfo>(new FakeClusterRoleInfo());
extra?.Invoke(services);
services.AddAkka("otopcua-test", (ab, _) =>
{
ab.AddHocon(@"
akka.actor.provider = ""Akka.Cluster.ClusterActorRefProvider, Akka.Cluster""
akka.remote.dot-netty.tcp.hostname = ""127.0.0.1""
akka.remote.dot-netty.tcp.port = 0
akka.cluster.seed-nodes = []
akka.cluster.roles = [""driver""]
", HoconAddMode.Prepend);
ab.WithOtOpcUaRuntimeActors();
});
})
.Build();
/// <summary>Non-throwing fake value writer that acks every batch.</summary>
private sealed class FakeValueWriter : IHistorianValueWriter
{
/// <summary>Acks the write unconditionally.</summary>
public Task<bool> WriteLiveValuesAsync(
string tag, IReadOnlyList<HistorizationValue> values, CancellationToken ct)
=> Task.FromResult(true);
}
/// <summary>Empty in-memory outbox fake — the spawn test only needs construction, not draining.</summary>
private sealed class FakeOutbox : IHistorizationOutbox
{
/// <summary>Never drops (unbounded).</summary>
public long DroppedCount => 0;
/// <summary>No-op append.</summary>
public ValueTask AppendAsync(HistorizationOutboxEntry entry, CancellationToken ct) => ValueTask.CompletedTask;
/// <summary>Always returns an empty batch.</summary>
public ValueTask<IReadOnlyList<HistorizationOutboxEntry>> PeekBatchAsync(int max, CancellationToken ct)
=> ValueTask.FromResult<IReadOnlyList<HistorizationOutboxEntry>>(Array.Empty<HistorizationOutboxEntry>());
/// <summary>No-op remove.</summary>
public ValueTask RemoveAsync(Guid id, CancellationToken ct) => ValueTask.CompletedTask;
/// <summary>Always empty.</summary>
public ValueTask<int> CountAsync(CancellationToken ct) => ValueTask.FromResult(0);
/// <summary>No-op dispose.</summary>
public void Dispose() { }
}
/// <summary>In-memory database factory for testing.</summary>
private sealed class InMemoryConfigDbFactory(string dbName) : IDbContextFactory<OtOpcUaConfigDbContext>
{
/// <summary>Creates a new in-memory database context.</summary>
public OtOpcUaConfigDbContext CreateDbContext()
{
var opts = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
.UseInMemoryDatabase(dbName)
.Options;
return new OtOpcUaConfigDbContext(opts);
}
}
/// <summary>Fake cluster role information for testing.</summary>
private sealed class FakeClusterRoleInfo : IClusterRoleInfo
{
/// <summary>Gets the local node ID.</summary>
public NodeId LocalNode { get; } = NodeId.Parse("test-node");
/// <summary>Gets the local roles.</summary>
public IReadOnlySet<string> LocalRoles { get; } = new HashSet<string>(["driver"]);
/// <summary>Gets the local node's cluster-scoped role. None configured in this fake.</summary>
public string? ClusterRole => null;
/// <summary>Gets the ClusterId extracted from <see cref="ClusterRole"/>. None configured in this fake.</summary>
public string? ClusterId => null;
/// <summary>Determines whether the local node has the specified role.</summary>
/// <param name="role">The role to check.</param>
public bool HasRole(string role) => LocalRoles.Contains(role);
/// <summary>Gets the members with the specified role.</summary>
/// <param name="role">The role to query.</param>
public IReadOnlyList<NodeId> MembersWithRole(string role) => Array.Empty<NodeId>();
/// <summary>Gets the leader node for the specified role.</summary>
/// <param name="role">The role to query.</param>
public NodeId? RoleLeader(string role) => null;
/// <summary>Raised when the role leader changes.</summary>
public event EventHandler<RoleLeaderChangedEventArgs>? RoleLeaderChanged
{
add { _ = value; }
remove { _ = value; }
}
}
}