Files
scadaproj/ZB.MOM.WW.Secrets/tests/ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests/DependencyInjection/HostedProcessResolutionTests.cs
T
Joseph Doherty a1df7ef26c fix(secrets): root-cause + fix the Akka replicator's hosted-process DI deadlock (0.2.2)
Closes the defect in scadaproj#1. The hang was never Akka: the package's DI
wiring closed a circular singleton dependency the container cannot see through
factory lambdas — ISecretStore (ReplicatingSecretStore decorator) ->
ISecretReplicator -> SecretReplicationActorProvider -> ISecretCacheInvalidator
-> DefaultSecretResolver -> ISecretStore. Resolution recurses around the loop
until MS.DI's StackGuard hops it onto a fresh thread-pool thread, which then
blocks forever on a singleton call-site lock the first thread still holds:
a silent permanent hang instead of a stack overflow. Managed stacks from
dotnet-dump show the repeating cycle and both parked threads; both candidate
causes in the issue (DistributedPubSub.Get vs the Lazy lock, missing
Akka.Cluster.Tools HOCON) are disproven — the actor constructor was never
reached, and the deadlock reproduces on a single non-clustered node.

Fix: defer the one cycle-closing edge. The provider now gets a
DeferredSecretCacheInvalidator that resolves the real invalidator on first
eviction — which only happens when a replicated row is applied, strictly after
graph resolution. Severing the edge instead is wrong: a null-invalidator
experiment ran the live gate at 5/6, with deleted secrets still resolving on
the peer. The SqlServer package never had the cycle (its replicator chain
never touches the invalidator), which is why the hub gate always passed.

Verified: live 2-node convergence gate now 6/6 (was: infinite hang), including
the delete-visibility check that proves the deferred invalidator really evicts.
New HostedProcessResolutionTests builds the graph as a host does (container-
registered ActorSystem, hosted services, watchdogged resolves) and fails on
0.2.1; DeferredSecretCacheInvalidatorTests pins the wrapper contract. Full
suite 180 passed / 0 failed / 15 skipped (env-gated live SQL).

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 14:39:40 -04:00

154 lines
6.7 KiB
C#

using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using ZB.MOM.WW.Secrets.Abstractions;
using ZB.MOM.WW.Secrets.Replication;
using ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.DependencyInjection;
namespace ZB.MOM.WW.Secrets.Replicator.AkkaDotNet.Tests.DependencyInjection;
/// <summary>
/// Builds the replication graph the way a real application host does — container-registered
/// <see cref="ActorSystem"/>, hosted services started, services resolved from the built provider —
/// and asserts resolution actually completes.
/// </summary>
/// <remarks>
/// <para>
/// This is the regression test for scadaproj#1. The 0.2.x graph had a circular singleton
/// dependency reachable only through the container: <c>ISecretStore</c> (the
/// <c>ReplicatingSecretStore</c> decorator) → <c>ISecretReplicator</c> →
/// <c>SecretReplicationActorProvider</c> → <c>ISecretCacheInvalidator</c> →
/// <c>DefaultSecretResolver</c> → <c>ISecretStore</c>. Every edge is an opaque factory lambda, so
/// the container cannot detect the cycle statically; at runtime the resolution recurses until
/// MS.DI's <c>StackGuard</c> moves it onto a fresh thread-pool thread, which then blocks forever on
/// a singleton call-site lock the first thread still holds. The symptom was a silent, permanent
/// hang — not a stack overflow, not an exception — which is why every direct-construction test
/// passed while any hosted process froze at startup.
/// </para>
/// <para>
/// The suite's other cluster tests construct the actor with <c>system.ActorOf</c> directly and can
/// never see this defect. Nothing here may resolve the affected services before the assertions do.
/// </para>
/// </remarks>
public sealed class HostedProcessResolutionTests : IAsyncLifetime
{
// Generous next to the milliseconds a healthy resolve takes, tiny next to CI noise: on
// regression the resolve NEVER completes, so the only cost of margin is this test's runtime.
private static readonly TimeSpan ResolveBudget = TimeSpan.FromSeconds(20);
private readonly string _dbPath =
Path.Combine(Path.GetTempPath(), $"zb-hosted-di-{Guid.NewGuid():N}.db");
private ActorSystem _system = null!;
private ServiceProvider _provider = null!;
public async Task InitializeAsync()
{
// A single self-joined cluster node: `actor.provider = cluster` is what the replication
// actor's DistributedPubSub mediator needs, and joining makes the rig an honest miniature
// of an app node rather than a special case.
_system = ActorSystem.Create("zb-secrets-cluster", ConfigurationFactory.ParseString("""
akka {
loglevel = WARNING
actor.provider = cluster
remote.dot-netty.tcp {
hostname = "127.0.0.1"
public-hostname = "127.0.0.1"
port = 0
}
}
""").WithFallback(AkkaSecretsReplication.SerializationConfig));
Cluster cluster = Cluster.Get(_system);
cluster.Join(cluster.SelfAddress);
DateTime deadline = DateTime.UtcNow + TimeSpan.FromSeconds(20);
while (cluster.State.Members.All(m => m.Status != MemberStatus.Up))
{
if (DateTime.UtcNow > deadline)
{
throw new TimeoutException("Single-node cluster never reached Up.");
}
await Task.Delay(100);
}
IConfigurationRoot config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["Secrets:SqlitePath"] = _dbPath,
["Secrets:RunMigrationsOnStartup"] = "true",
["Secrets:Replication:AnnounceInterval"] = "00:00:30",
})
.Build();
var services = new ServiceCollection();
services.AddLogging();
services.AddSingleton(_system);
services.AddZbSecretsAkkaReplication(config, "Secrets", "Secrets:Replication");
_provider = services.BuildServiceProvider();
// Started exactly as a host would, BEFORE anything resolves the replicated store — the
// order OtOpcUa's startup follows, and the order under which the deadlock was reproduced.
foreach (IHostedService hosted in _provider.GetServices<IHostedService>())
{
await hosted.StartAsync(CancellationToken.None);
}
}
[Fact]
public async Task Resolving_the_replicated_graph_through_the_container_completes()
{
// One test covers all three entry points IN this order: the store is the edge OtOpcUa's
// startup hook actually resolves, the replicator is the narrowest reproduction of the
// hang, and the resolver closes the loop from the invalidator's side. Split into separate
// facts, each would rebuild the rig only to hit the same three singletons.
ISecretStore store = await ResolveWithinBudgetAsync(
sp => sp.GetRequiredService<ISecretStore>(), nameof(ISecretStore));
ISecretReplicator replicator = await ResolveWithinBudgetAsync(
sp => sp.GetRequiredService<ISecretReplicator>(), nameof(ISecretReplicator));
ISecretResolver resolver = await ResolveWithinBudgetAsync(
sp => sp.GetRequiredService<ISecretResolver>(), nameof(ISecretResolver));
Assert.IsType<ReplicatingSecretStore>(store);
Assert.IsType<AkkaSecretReplicator>(replicator);
Assert.NotNull(resolver);
}
private async Task<T> ResolveWithinBudgetAsync<T>(
Func<IServiceProvider, T> resolve, string label)
{
// The failure mode under test is an infinite wait inside the container, so the resolve
// runs on the pool with a watchdog. On regression the worker thread is unrecoverable —
// acceptable in a test process that is about to exit, and vastly better than hanging CI.
Task<T> resolution = Task.Run(() => resolve(_provider));
Task first = await Task.WhenAny(resolution, Task.Delay(ResolveBudget));
Assert.True(
first == resolution,
$"Resolving {label} did not complete within {ResolveBudget.TotalSeconds:0}s — " +
"the DI singleton cycle deadlock (scadaproj#1) has regressed.");
return await resolution;
}
public async Task DisposeAsync()
{
await _provider.DisposeAsync();
await _system.Terminate();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (string path in new[] { _dbPath, _dbPath + "-wal", _dbPath + "-shm" })
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}