Joseph Doherty
250 lines
12 KiB
C#
250 lines
12 KiB
C#
using System.Diagnostics.CodeAnalysis;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Hosting;
|
|
using Akka.Hosting;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Hosting;
|
|
using Microsoft.Extensions.Options;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.OtOpcUa.Cluster;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Interfaces;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
|
using ZB.MOM.WW.Secrets.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
|
|
|
/// <summary>
|
|
/// Guards the one thing every other test in this project structurally cannot see: whether an
|
|
/// admin node <b>boots</b> with the real registration graph behind the mesh command boundary.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// The Phase 1A live gate failed here. <c>WithOtOpcUaControlPlaneSingletons</c>'s first
|
|
/// starter resolved <c>IMeshCommandDialer</c> and <c>IMeshPskProvider</c> eagerly, and their
|
|
/// graph reaches back into the actor registry — dialer → <c>IMeshPskProvider</c> →
|
|
/// <c>ISecretResolver</c> → <c>AddZbSecrets</c> → <c>IAuditWriter</c> →
|
|
/// <c>ActorAuditWriter</c>, whose constructor calls
|
|
/// <c>registry.Get<AuditWriterActorKey>()</c> for a singleton registered further down
|
|
/// the SAME builder. Every central node died on
|
|
/// <c>MissingActorRegistryEntryException</c>, in every transport mode, before serving
|
|
/// anything.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Why the existing tests were blind.</b> <c>MeshGrpcCommandDialerTests</c> activates the
|
|
/// dialer from a <c>ServiceCollection</c> that has no secrets registration at all, so its
|
|
/// graph stops one hop in; the <c>CentralCommunicationActorTests</c> facts inject fakes
|
|
/// directly and never touch DI; and the existing whole-host boots register no mesh provider,
|
|
/// so the eager lookups returned <see langword="null"/> and resolved nothing. Every one of
|
|
/// them passed while no central node could start.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>So the assertion is <c>StartAsync</c> itself.</b> The failure happens while the Akka
|
|
/// builder's starters execute, which is inside <c>StartAsync</c> — building the
|
|
/// <c>ServiceProvider</c> and resolving from it by hand would not reproduce it. The graph
|
|
/// below is the production one from <c>IMeshCommandDialer</c> down to the registry-bound
|
|
/// audit writer; only the secret store itself is a stand-in, because a real one would add a
|
|
/// KEK and a SQLite file to a test whose subject is starter ORDER.
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class ControlPlaneStarterOrderTests
|
|
{
|
|
[Theory]
|
|
[InlineData(MeshTransportOptions.ModeDps)]
|
|
[InlineData(MeshTransportOptions.ModeClusterClient)]
|
|
[InlineData(MeshTransportOptions.ModeGrpc)]
|
|
public async Task An_admin_node_boots_with_the_real_mesh_dependency_graph(string mode)
|
|
{
|
|
// Every mode, because the eager resolve was mode-independent: the starter resolved both
|
|
// dependencies before anything looked at MeshTransport:Mode, so Dps — the DEFAULT, and the
|
|
// rollback target — died exactly like Grpc did.
|
|
using var host = BuildAdminHost(mode);
|
|
|
|
await Should.NotThrowAsync(() => host.StartAsync());
|
|
|
|
try
|
|
{
|
|
var registry = host.Services.GetRequiredService<ActorRegistry>();
|
|
registry.TryGet<CentralCommunicationActorKey>(out _).ShouldBeTrue(
|
|
"the command boundary must be registered on a booted admin node");
|
|
registry.TryGet<AuditWriterActorKey>(out _).ShouldBeTrue(
|
|
"the audit singleton the eager resolve reached for too early must exist by now");
|
|
}
|
|
finally
|
|
{
|
|
await host.StopAsync();
|
|
}
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(MeshTransportOptions.ModeDps)]
|
|
[InlineData(MeshTransportOptions.ModeClusterClient)]
|
|
[InlineData(MeshTransportOptions.ModeGrpc)]
|
|
public async Task The_deferred_dependencies_are_resolved_and_used_once_the_host_is_up(string mode)
|
|
{
|
|
// Deferring must not become "never resolved" — that would trade a loud boot failure for a
|
|
// silent one, with the known-cluster gate permanently unpublished and the dialer permanently
|
|
// absent. So this asserts the far end of the deferral: the actor resolved the PSK provider
|
|
// FROM THE CONTAINER and drove it, through the ordinary contact-refresh path. A Theory over
|
|
// all three modes because publication is mode-independent (the gate follows the endpoint).
|
|
//
|
|
// The provider is registered as a TYPE with an ISecretResolver constructor, so the resolve
|
|
// walks the same registry-bound chain the production MeshPskProvider does — which makes
|
|
// this fact the race detector for the PreStart-vs-starter-#4 window: a lost race that
|
|
// stayed null forever would never publish, and the TaskCompletionSource below would time
|
|
// out red. (Memoize-on-first-use in the actor is what heals the lost race at the first
|
|
// RefreshContacts message.)
|
|
//
|
|
// The signal is a TaskCompletionSource rather than a counter, because PreStart runs on the
|
|
// actor's own thread and the contact load is a piped DB read: there is no instant after
|
|
// StartAsync at which a synchronous assertion is correct.
|
|
using var host = BuildAdminHost(mode, useSignallingProvider: true);
|
|
|
|
await host.StartAsync();
|
|
|
|
try
|
|
{
|
|
var signal = (SignallingPskProvider)host.Services.GetRequiredService<IMeshPskProvider>();
|
|
await signal.Published.WaitAsync(TimeSpan.FromSeconds(20));
|
|
}
|
|
finally
|
|
{
|
|
await host.StopAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Builds an admin host through the production cluster bootstrap and the production control-plane
|
|
/// starter registration, with the mesh dependency graph wired as it is on a real central node.
|
|
/// </summary>
|
|
/// <param name="mode">The mesh transport mode to boot under.</param>
|
|
/// <param name="useSignallingProvider">
|
|
/// Substitutes <see cref="SignallingPskProvider"/> for the real <see cref="MeshPskProvider"/>.
|
|
/// Only the resolution test sets it; the boot test keeps the production type so the graph it
|
|
/// walks is the production graph. The signalling provider is registered as a factory taking
|
|
/// <see cref="ISecretResolver"/> from the container, so its resolution walks the same
|
|
/// registry-bound chain either way.
|
|
/// </param>
|
|
/// <returns>The unstarted host.</returns>
|
|
private static IHost BuildAdminHost(string mode, bool useSignallingProvider = false)
|
|
{
|
|
var clusterOptions = new AkkaClusterOptions
|
|
{
|
|
Port = 0,
|
|
Hostname = "127.0.0.1",
|
|
PublicHostname = "127.0.0.1",
|
|
SeedNodes = [],
|
|
Roles = ["admin"],
|
|
};
|
|
|
|
return Host.CreateDefaultBuilder()
|
|
.ConfigureServices(services =>
|
|
{
|
|
services.AddSingleton<IOptions<AkkaClusterOptions>>(Options.Create(clusterOptions));
|
|
services.AddSingleton<IOptions<MeshTransportOptions>>(Options.Create(
|
|
new MeshTransportOptions { Mode = mode, DispatchTimeoutSeconds = 10 }));
|
|
services.AddDbContextFactory<OtOpcUaConfigDbContext>(
|
|
o => o.UseInMemoryDatabase($"starter-order-{Guid.NewGuid():N}"));
|
|
services.AddSingleton<IClusterRoleInfo>(new StubClusterRoleInfo());
|
|
|
|
// THE GRAPH. Production types from the dialer down to the registry-bound writer; the
|
|
// order of these registrations is deliberately irrelevant, because the defect was
|
|
// never about registration order — it was about WHEN the graph is first walked.
|
|
services.AddSingleton<IAuditWriter, RegistryBoundAuditWriter>();
|
|
services.AddSingleton<ISecretResolver, AuditingSecretResolver>();
|
|
if (useSignallingProvider)
|
|
services.AddSingleton<IMeshPskProvider>(sp =>
|
|
new SignallingPskProvider(sp.GetRequiredService<ISecretResolver>()));
|
|
else
|
|
services.AddSingleton<IMeshPskProvider, MeshPskProvider>();
|
|
services.AddSingleton<IMeshCommandDialer, MeshGrpcCommandDialer>();
|
|
|
|
services.AddAkka("otopcua-starter-order-test", (ab, sp) =>
|
|
{
|
|
ab.WithOtOpcUaClusterBootstrap(sp);
|
|
ab.WithOtOpcUaControlPlaneSingletons();
|
|
});
|
|
})
|
|
.Build();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stands in for <c>ActorAuditWriter</c>: an <see cref="IAuditWriter"/> that resolves a
|
|
/// cluster singleton from the <see cref="ActorRegistry"/> <b>in its constructor</b>. That
|
|
/// constructor is the tripwire — it throws
|
|
/// <c>MissingActorRegistryEntryException</c> for exactly as long as the audit singleton has
|
|
/// not been registered yet.
|
|
/// </summary>
|
|
private sealed class RegistryBoundAuditWriter : IAuditWriter
|
|
{
|
|
private readonly IActorRef _proxy;
|
|
|
|
public RegistryBoundAuditWriter(ActorRegistry registry) => _proxy = registry.Get<AuditWriterActorKey>();
|
|
|
|
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
|
{
|
|
_proxy.Tell(evt);
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// An <see cref="IMeshPskProvider"/> that completes a task the first time the actor publishes
|
|
/// the known-cluster set. Takes <see cref="ISecretResolver"/> it never uses, purely so that
|
|
/// resolving it still walks the same registry-bound chain the real provider does.
|
|
/// </summary>
|
|
private sealed class SignallingPskProvider : IMeshPskProvider
|
|
{
|
|
private readonly TaskCompletionSource _published =
|
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public SignallingPskProvider(ISecretResolver resolver) => ArgumentNullException.ThrowIfNull(resolver);
|
|
|
|
/// <summary>Completes once <see cref="SetKnownClusters"/> has been called.</summary>
|
|
public Task Published => _published.Task;
|
|
|
|
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
|
|
{
|
|
psk = null;
|
|
return false;
|
|
}
|
|
|
|
public bool Matches(string clusterId, string presented) => false;
|
|
|
|
public void SetKnownClusters(IReadOnlyCollection<string> clusterIds) => _published.TrySetResult();
|
|
|
|
public void Invalidate(string clusterId) { }
|
|
|
|
public void InvalidateAll() { }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stands in for the shared secret store's resolver. Its only load-bearing property is the
|
|
/// one that broke boot: it depends on <see cref="IAuditWriter"/>, exactly as
|
|
/// <c>AddZbSecrets</c>' resolver registration does.
|
|
/// </summary>
|
|
private sealed class AuditingSecretResolver : ISecretResolver
|
|
{
|
|
public AuditingSecretResolver(IAuditWriter auditWriter) => ArgumentNullException.ThrowIfNull(auditWriter);
|
|
|
|
public Task<string?> GetAsync(SecretName name, CancellationToken ct) => Task.FromResult<string?>(null);
|
|
}
|
|
|
|
/// <summary>Minimal <see cref="IClusterRoleInfo"/>; the admin singleton factories read it.</summary>
|
|
private sealed class StubClusterRoleInfo : IClusterRoleInfo
|
|
{
|
|
public string? ClusterRole => null;
|
|
public string? ClusterId => null;
|
|
public NodeId LocalNode => NodeId.Parse("127.0.0.1:0");
|
|
public IReadOnlySet<string> LocalRoles => new HashSet<string>();
|
|
public bool HasRole(string role) => false;
|
|
public IReadOnlyList<NodeId> MembersWithRole(string role) => [];
|
|
public NodeId? RoleLeader(string role) => null;
|
|
public event EventHandler<RoleLeaderChangedEventArgs>? RoleLeaderChanged { add { } remove { } }
|
|
}
|
|
}
|