test(host): 2-node integration test harness + consolidate to one ActorSystem (Task 58)
Builds TwoNodeClusterHarness: two in-process Host-equivalent nodes sharing
an in-memory ConfigDb. Forms a 2-member Akka cluster. ClusterFormationTests
proves both nodes see each other as admin+driver role members.
Fixes a real production bug uncovered while wiring the harness — Program.cs
ran two separate ActorSystems (one from AddOtOpcUaCluster.AkkaHostedService
with cluster HOCON, one from Akka.Hosting.AddAkka with bare HOCON). Cluster
singletons landed on the bare ActorSystem and could not actually form a
cluster ("Configuration does not contain `akka.cluster` node").
Consolidation:
- AddOtOpcUaCluster now only binds AkkaClusterOptions + registers IClusterRoleInfo
- New WithOtOpcUaClusterBootstrap pushes embedded HOCON + Remote/Cluster options
into Akka.Hosting's AkkaConfigurationBuilder
- AkkaHostedService.cs deleted — Akka.Hosting now owns the lifecycle
- Program.cs + harness call WithOtOpcUaClusterBootstrap inside AddAkka
Why not WebApplicationFactory<Program>? Program.cs reads OTOPCUA_ROLES from
process env (shared across in-process WAFs); the harness replays Program.cs's
DI graph from a clean WebApplicationBuilder per node with per-node config
overrides. Same production extensions, isolated config + Kestrel + Akka ports.
Tests: 93 v2 tests pass (was 91 + 2 new cluster formation), 0 skipped.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,43 @@
|
||||
using Akka.Cluster;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Smoke test: verifies <see cref="TwoNodeClusterHarness"/> boots two nodes and they form
|
||||
/// a 2-member cluster with the expected role topology. Failover + deploy scenarios layer
|
||||
/// on top in Task 59.
|
||||
/// </summary>
|
||||
public sealed class ClusterFormationTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Two_nodes_form_a_2_member_cluster()
|
||||
{
|
||||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||||
|
||||
var aCluster = Akka.Cluster.Cluster.Get(harness.NodeASystem);
|
||||
var bCluster = Akka.Cluster.Cluster.Get(harness.NodeBSystem);
|
||||
|
||||
aCluster.State.Members.Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
|
||||
bCluster.State.Members.Count(m => m.Status == MemberStatus.Up).ShouldBe(2);
|
||||
|
||||
var aRoles = aCluster.State.Members.SelectMany(m => m.Roles).Distinct().ToHashSet();
|
||||
aRoles.ShouldContain("admin");
|
||||
aRoles.ShouldContain("driver");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Both_nodes_see_each_other_as_role_members()
|
||||
{
|
||||
await using var harness = await TwoNodeClusterHarness.StartAsync();
|
||||
|
||||
var aCluster = Akka.Cluster.Cluster.Get(harness.NodeASystem);
|
||||
aCluster.State.Members
|
||||
.Where(m => m.Roles.Contains("driver") && m.Status == MemberStatus.Up)
|
||||
.Count().ShouldBe(2);
|
||||
aCluster.State.Members
|
||||
.Where(m => m.Roles.Contains("admin") && m.Status == MemberStatus.Up)
|
||||
.Count().ShouldBe(2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
using System.Net.Sockets;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Akka.Hosting;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Clients;
|
||||
using ZB.MOM.WW.OtOpcUa.AdminUI.Hubs;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Configuration;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane;
|
||||
using ZB.MOM.WW.OtOpcUa.Host.Health;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime;
|
||||
using ZB.MOM.WW.OtOpcUa.Security;
|
||||
using ZB.MOM.WW.OtOpcUa.Security.Endpoints;
|
||||
using ZB.MOM.WW.OtOpcUa.Security.Ldap;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// Spins up two in-process <c>OtOpcUa.Host</c>-equivalent <see cref="WebApplication"/> instances
|
||||
/// that share an in-memory <see cref="OtOpcUaConfigDbContext"/> and form a 2-member Akka cluster.
|
||||
/// Both nodes carry the <c>admin</c> + <c>driver</c> roles, matching design §8's failover-test
|
||||
/// 2-node profile.
|
||||
///
|
||||
/// Why not <c>WebApplicationFactory<Program></c>?
|
||||
/// Program.cs reads <c>OTOPCUA_ROLES</c> from process env (shared across in-process WAF
|
||||
/// instances) and writes both Serilog file sinks + Akka cluster TCP listener to the host
|
||||
/// process — neither survives two parallel WAFs cleanly. This harness instead replays the
|
||||
/// Program.cs DI graph from a clean <see cref="WebApplicationBuilder"/> per node with
|
||||
/// per-node config overrides. The production wiring is the same set of extensions
|
||||
/// (<see cref="ServiceCollectionExtensions.AddOtOpcUaConfigDb"/>,
|
||||
/// <see cref="AkkaCluster.ServiceCollectionExtensions.AddOtOpcUaCluster"/>,
|
||||
/// <see cref="AddOtOpcUaAuth"/>, <see cref="AddOtOpcUaHealth"/>,
|
||||
/// <see cref="WithOtOpcUaControlPlaneSingletons"/>,
|
||||
/// <see cref="WithOtOpcUaRuntimeActors"/>).
|
||||
/// </summary>
|
||||
public sealed class TwoNodeClusterHarness : IAsyncDisposable
|
||||
{
|
||||
public const string TestRoles = "admin,driver";
|
||||
public static readonly string SharedDbName = $"two-node-cluster-{Guid.NewGuid():N}";
|
||||
|
||||
public WebApplication NodeA { get; private set; } = null!;
|
||||
public WebApplication NodeB { get; private set; } = null!;
|
||||
|
||||
public int NodeAAkkaPort { get; private set; }
|
||||
public int NodeBAkkaPort { get; private set; }
|
||||
|
||||
public ActorSystem NodeASystem => NodeA.Services.GetRequiredService<ActorSystem>();
|
||||
public ActorSystem NodeBSystem => NodeB.Services.GetRequiredService<ActorSystem>();
|
||||
|
||||
/// <summary>Boots both nodes and waits up to <paramref name="formationTimeout"/> for cluster convergence.</summary>
|
||||
public static async Task<TwoNodeClusterHarness> StartAsync(TimeSpan? formationTimeout = null)
|
||||
{
|
||||
var harness = new TwoNodeClusterHarness();
|
||||
harness.NodeAAkkaPort = AllocateFreePort();
|
||||
harness.NodeBAkkaPort = AllocateFreePort();
|
||||
|
||||
// Node A boots first as the seed.
|
||||
harness.NodeA = await BuildNodeAsync(
|
||||
akkaPort: harness.NodeAAkkaPort,
|
||||
seedAkkaPort: harness.NodeAAkkaPort,
|
||||
dbName: SharedDbName);
|
||||
|
||||
harness.NodeB = await BuildNodeAsync(
|
||||
akkaPort: harness.NodeBAkkaPort,
|
||||
seedAkkaPort: harness.NodeAAkkaPort,
|
||||
dbName: SharedDbName);
|
||||
|
||||
await WaitForClusterFormationAsync(
|
||||
harness.NodeASystem,
|
||||
harness.NodeBSystem,
|
||||
formationTimeout ?? TimeSpan.FromSeconds(20));
|
||||
|
||||
return harness;
|
||||
}
|
||||
|
||||
private static async Task<WebApplication> BuildNodeAsync(int akkaPort, int seedAkkaPort, string dbName)
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder(new WebApplicationOptions { Args = [] });
|
||||
|
||||
builder.WebHost.UseKestrel(o => o.Listen(System.Net.IPAddress.Loopback, 0));
|
||||
builder.Configuration.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["ConnectionStrings:ConfigDb"] = "Server=test;Database=test;Trusted_Connection=True;TrustServerCertificate=True;",
|
||||
["Cluster:Hostname"] = "127.0.0.1",
|
||||
["Cluster:Port"] = akkaPort.ToString(),
|
||||
["Cluster:PublicHostname"] = "127.0.0.1",
|
||||
["Cluster:SeedNodes:0"] = $"akka.tcp://otopcua@127.0.0.1:{seedAkkaPort}",
|
||||
["Cluster:Roles:0"] = "admin",
|
||||
["Cluster:Roles:1"] = "driver",
|
||||
["Security:Jwt:SigningKey"] = "two-node-harness-test-signing-key-with-enough-bytes-for-hs256",
|
||||
["Security:Jwt:Issuer"] = "otopcua-test",
|
||||
["Security:Jwt:Audience"] = "otopcua-test",
|
||||
});
|
||||
|
||||
// Replicate Program.cs role wiring with the harness-shared in-memory ConfigDb.
|
||||
builder.Services.AddDbContextFactory<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
|
||||
builder.Services.AddDbContext<OtOpcUaConfigDbContext>(opt => opt.UseInMemoryDatabase(dbName));
|
||||
builder.Services.AddOtOpcUaCluster(builder.Configuration);
|
||||
|
||||
builder.Services.AddAkka("otopcua", (ab, sp) =>
|
||||
{
|
||||
ab.WithOtOpcUaClusterBootstrap(sp);
|
||||
ab.WithOtOpcUaControlPlaneSingletons();
|
||||
ab.WithOtOpcUaRuntimeActors();
|
||||
});
|
||||
|
||||
builder.Services.AddOtOpcUaAuth(builder.Configuration);
|
||||
builder.Services.AddSingleton<ILdapAuthService, StubLdapAuthService>();
|
||||
builder.Services.AddAdminUI();
|
||||
builder.Services.AddSignalR();
|
||||
builder.Services.AddOtOpcUaAdminClients();
|
||||
builder.Services.AddOtOpcUaHealth();
|
||||
|
||||
var app = builder.Build();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
app.MapOtOpcUaAuth();
|
||||
app.MapOtOpcUaHubs();
|
||||
app.MapOtOpcUaHealth();
|
||||
|
||||
await app.StartAsync();
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task WaitForClusterFormationAsync(ActorSystem a, ActorSystem b, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var aMembers = Akka.Cluster.Cluster.Get(a).State.Members
|
||||
.Where(m => m.Status == MemberStatus.Up).ToArray();
|
||||
var bMembers = Akka.Cluster.Cluster.Get(b).State.Members
|
||||
.Where(m => m.Status == MemberStatus.Up).ToArray();
|
||||
if (aMembers.Length >= 2 && bMembers.Length >= 2) return;
|
||||
await Task.Delay(200);
|
||||
}
|
||||
throw new TimeoutException(
|
||||
$"Cluster did not form within {timeout}. " +
|
||||
$"A up={Akka.Cluster.Cluster.Get(a).State.Members.Count(m => m.Status == MemberStatus.Up)}, " +
|
||||
$"B up={Akka.Cluster.Cluster.Get(b).State.Members.Count(m => m.Status == MemberStatus.Up)}");
|
||||
}
|
||||
|
||||
private static int AllocateFreePort()
|
||||
{
|
||||
using var listener = new TcpListener(System.Net.IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
var port = ((System.Net.IPEndPoint)listener.LocalEndpoint).Port;
|
||||
listener.Stop();
|
||||
return port;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (NodeB is not null) await NodeB.DisposeAsync();
|
||||
if (NodeA is not null) await NodeA.DisposeAsync();
|
||||
}
|
||||
|
||||
private sealed class StubLdapAuthService : ILdapAuthService
|
||||
{
|
||||
public Task<LdapAuthResult> AuthenticateAsync(string username, string password, CancellationToken ct = default)
|
||||
=> Task.FromResult(new LdapAuthResult(
|
||||
Success: password == "valid-password",
|
||||
DisplayName: username,
|
||||
Username: username,
|
||||
Groups: ["FleetAdmin"],
|
||||
Roles: ["FleetAdmin"],
|
||||
Error: null));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<IsTestProject>true</IsTestProject>
|
||||
<RootNamespace>ZB.MOM.WW.OtOpcUa.Host.IntegrationTests</RootNamespace>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="xunit.v3"/>
|
||||
<PackageReference Include="Shouldly"/>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing"/>
|
||||
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory"/>
|
||||
<PackageReference Include="Akka.Hosting"/>
|
||||
<PackageReference Include="xunit.runner.visualstudio">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\..\src\Server\ZB.MOM.WW.OtOpcUa.Host\ZB.MOM.WW.OtOpcUa.Host.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Cluster\ZB.MOM.WW.OtOpcUa.Cluster.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Configuration\ZB.MOM.WW.OtOpcUa.Configuration.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Core\ZB.MOM.WW.OtOpcUa.Commons\ZB.MOM.WW.OtOpcUa.Commons.csproj"/>
|
||||
<ProjectReference Include="..\..\..\src\Server\ZB.MOM.WW.OtOpcUa.Security\ZB.MOM.WW.OtOpcUa.Security.csproj"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<NuGetAuditSuppress Include="https://github.com/advisories/GHSA-g94r-2vxg-569j"/>
|
||||
<NuGetAuditSuppress Include="https://github.com/advisories/GHSA-h958-fxgg-g7w3"/>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user