using System.Diagnostics;
using System.Net;
using Akka.Actor;
using Akka.Cluster;
using Akka.Configuration;
using Akka.Hosting;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Hosting.Server;
using Microsoft.AspNetCore.Hosting.Server.Features;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Shouldly;
using Xunit;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.LocalDb.Replication;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Host.Configuration;
using AkkaCluster = Akka.Cluster.Cluster;
namespace ZB.MOM.WW.OtOpcUa.Host.IntegrationTests.LocalDb;
///
/// lmxopcua#906 — a graceful stop of a node whose peer holds an open LocalDb sync stream must leave
/// the Akka cluster FIRST and finish promptly, not wait out Kestrel's graceful-drain timeout.
///
///
///
/// The passive node is a real — the hosting model whose hosted-service
/// order caused the bug (the web server stops before Akka) — with a real Kestrel h2c listener, the
/// real MapZbLocalDbSync behind the real auth interceptor, a real single-member Akka cluster
/// and the production
/// + . The peer is a real
/// SyncBackgroundService dialling it over loopback.
///
///
/// The web server's is wrapped to timestamp the moment its
/// StopAsync (the drain) begins, so "left the cluster before the drain" is asserted as an
/// ORDER, not inferred from a total duration.
///
///
/// The second test is the positive control: the same stack WITHOUT the fix reproduces #906 — the
/// stop waits out the whole shutdown timeout and the cluster is left only after the drain. Without
/// it, a fast stop in the first test could be a harness that never held a stream open at all.
///
///
[Collection("LocalDbPairConvergence")]
public sealed class GracefulStopWithOpenSyncStreamTests
{
private const string ApiKey = "otopcua-906-graceful-stop-key";
/// The host's ShutdownTimeout in these tests — Kestrel's drain budget (30 s in production).
private static readonly TimeSpan ShutdownTimeout = TimeSpan.FromSeconds(8);
static GracefulStopWithOpenSyncStreamTests() =>
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
[Fact]
public async Task GracefulStop_WithOpenSyncStream_LeavesClusterBeforeDrain_AndFinishesPromptly()
{
var result = await RunStopScenarioAsync(withFix: true);
// Left the cluster, and did so before the web server began draining.
result.MemberRemovedAt.ShouldNotBeNull("the stopping node never left the Akka cluster");
result.MemberRemovedAt.Value.ShouldBeLessThan(result.ServerStopBeganAt!.Value,
"the node must leave the cluster BEFORE Kestrel drains, so the peer promotes promptly");
// Finished well inside the drain budget: the open sync stream did not hold the stop.
result.StopElapsed.ShouldBeLessThan(TimeSpan.FromSeconds(4),
$"stop took {result.StopElapsed.TotalMilliseconds:0} ms against a {ShutdownTimeout.TotalSeconds:0} s drain budget");
result.MemberRemovedAt.Value.ShouldBeLessThan(TimeSpan.FromSeconds(4));
}
[Fact]
public async Task PositiveControl_WithoutTheFix_StopWaitsOutTheDrain_AndLeavesTheClusterLast()
{
var result = await RunStopScenarioAsync(withFix: false);
// The #906 shape: Kestrel waits for the open stream until the shutdown timeout aborts it,
// and only then does Akka's own hosted service leave the cluster.
result.StopElapsed.ShouldBeGreaterThanOrEqualTo(ShutdownTimeout - TimeSpan.FromSeconds(1));
result.MemberRemovedAt.ShouldNotBeNull();
result.MemberRemovedAt.Value.ShouldBeGreaterThan(result.ServerStopBeganAt!.Value);
result.MemberRemovedAt.Value.ShouldBeGreaterThanOrEqualTo(ShutdownTimeout - TimeSpan.FromSeconds(1));
}
private sealed record StopResult(TimeSpan StopElapsed, TimeSpan? MemberRemovedAt, TimeSpan? ServerStopBeganAt);
private static async Task RunStopScenarioAsync(bool withFix)
{
var ct = TestContext.Current.CancellationToken;
var pathA = Path.Combine(Path.GetTempPath(), $"otopcua-906-A-{Guid.NewGuid():N}.db");
var pathB = Path.Combine(Path.GetTempPath(), $"otopcua-906-B-{Guid.NewGuid():N}.db");
await using var dbA = BuildDatabaseProvider(pathA);
await using var dbB = BuildDatabaseProvider(pathB);
var clock = new Stopwatch();
var serverStopBegan = new ServerStopProbe(clock);
IHost? initiator = null;
WebApplication? passive = null;
try
{
passive = BuildPassiveNode(dbB.GetRequiredService(), serverStopBegan, withFix);
await passive.StartAsync(ct);
var system = passive.Services.GetRequiredService();
var cluster = AkkaCluster.Get(system);
cluster.Join(cluster.SelfAddress);
await WaitUntilAsync(() => cluster.State.Members.Any(m => m.Status == MemberStatus.Up),
"the passive node to come Up as its own seed");
TimeSpan? memberRemovedAt = null;
cluster.RegisterOnMemberRemoved(() => memberRemovedAt ??= clock.Elapsed);
var address = passive.Services.GetRequiredService()
.Features.Get()!.Addresses.Single();
initiator = await StartInitiatorAsync(dbA.GetRequiredService(), address, ct);
// The peer's stream is open and holding a session on the passive node.
var passiveStatus = passive.Services.GetRequiredService();
await WaitUntilAsync(() => passiveStatus.Connected, "the peer's sync stream to be open on the passive node");
clock.Start();
await passive.StopAsync(CancellationToken.None);
var elapsed = clock.Elapsed;
TestContext.Current.TestOutputHelper?.WriteLine(
$"withFix={withFix}: stop {elapsed.TotalMilliseconds:0} ms; member removed at " +
$"{memberRemovedAt?.TotalMilliseconds:0} ms; web-server drain began at " +
$"{serverStopBegan.BeganAt?.TotalMilliseconds:0} ms (drain budget {ShutdownTimeout.TotalSeconds:0} s)");
return new StopResult(elapsed, memberRemovedAt, serverStopBegan.BeganAt);
}
finally
{
if (initiator is not null)
{
try { await initiator.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
initiator.Dispose();
}
if (passive is not null)
await passive.DisposeAsync();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (var path in new[] { pathA, pathB })
foreach (var suffix in new[] { "", "-wal", "-shm" })
try { File.Delete(path + suffix); } catch { /* best effort */ }
}
}
private static WebApplication BuildPassiveNode(ILocalDb db, ServerStopProbe probe, bool withFix)
{
var builder = WebApplication.CreateBuilder();
builder.Environment.ApplicationName = typeof(GracefulStopWithOpenSyncStreamTests).Assembly.GetName().Name!;
builder.Logging.ClearProviders();
builder.WebHost.ConfigureKestrel(k =>
k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
builder.Services.Configure(o => o.ShutdownTimeout = ShutdownTimeout);
builder.Services.AddGrpc(o =>
{
o.Interceptors.Add();
if (withFix)
o.Interceptors.Add();
});
builder.Services.AddSingleton(db);
builder.Services.AddZbLocalDbReplication(ReplicationConfig(peerAddress: null));
if (!withFix)
{
// From LocalDb 0.3.8 the library itself ends open sync streams on ApplicationStopping
// (scadaproj#57), which would hide the #906 drain in this control. Re-register the
// service WITHOUT the host lifetime, i.e. the 0.3.7 behaviour, so the control still
// reproduces the bug the host-side fix guards against.
builder.Services.Replace(ServiceDescriptor.Singleton(sp => new LocalDbSyncService(
sp.GetRequiredService(),
sp.GetRequiredService>(),
sp.GetRequiredService(),
sp.GetService(),
sp.GetService(),
lifetime: null)));
}
builder.Services.AddAkka("otopcua906", (ab, _) => ab.AddHocon(
ConfigurationFactory.ParseString(
"akka.remote.dot-netty.tcp.hostname = \"127.0.0.1\"\n"
+ "akka.remote.dot-netty.tcp.public-hostname = \"127.0.0.1\"\n"
+ "akka.remote.dot-netty.tcp.port = 0\n"
+ "akka.cluster.seed-nodes = []\n"
+ "akka.loglevel = \"WARNING\"\n")
.WithFallback(HoconLoader.LoadBaseConfig()),
HoconAddMode.Prepend));
if (withFix)
builder.Services.AddOtOpcUaGracefulClusterLeave();
// Wrap the web server so the start of its drain is timestamped.
var serverDescriptor = builder.Services.Last(d => d.ServiceType == typeof(IServer));
builder.Services.Remove(serverDescriptor);
builder.Services.AddSingleton(sp =>
{
var inner = serverDescriptor.ImplementationFactory is { } factory
? (IServer)factory(sp)
: (IServer)ActivatorUtilities.CreateInstance(sp, serverDescriptor.ImplementationType!);
return probe.Wrap(inner);
});
var app = builder.Build();
app.MapZbLocalDbSync();
return app;
}
private static async Task StartInitiatorAsync(ILocalDb db, string peerAddress, CancellationToken ct)
{
var config = ReplicationConfig(peerAddress);
var host = new HostBuilder()
.ConfigureServices(services =>
{
services.AddLogging();
services.AddSingleton(db);
services.AddZbLocalDbReplication(config);
})
.Build();
await host.StartAsync(ct);
return host;
}
private static IConfiguration ReplicationConfig(string? peerAddress)
{
var values = new Dictionary
{
["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
["LocalDb:Replication:ApiKey"] = ApiKey,
};
if (peerAddress is not null)
values["LocalDb:Replication:PeerAddress"] = peerAddress;
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
}
private static ServiceProvider BuildDatabaseProvider(string path)
{
var config = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary
{
["LocalDb:Path"] = path,
["LocalDb:Replication:ApiKey"] = ApiKey,
})
.Build();
return new ServiceCollection()
.AddZbLocalDb(config, db => LocalDbSetup.OnReady(db, config))
.BuildServiceProvider();
}
private static async Task WaitUntilAsync(Func condition, string because)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (DateTime.UtcNow < deadline)
{
if (condition())
return;
await Task.Delay(50, TestContext.Current.CancellationToken);
}
Assert.Fail($"Timed out waiting for: {because}");
}
/// Records when the wrapped web server's StopAsync (its graceful drain) begins.
private sealed class ServerStopProbe(Stopwatch clock)
{
public TimeSpan? BeganAt { get; private set; }
private void MarkBegan() => BeganAt ??= clock.Elapsed;
public IServer Wrap(IServer inner) => new RecordingServer(inner, this);
private sealed class RecordingServer(IServer inner, ServerStopProbe probe) : IServer
{
public IFeatureCollection Features => inner.Features;
public Task StartAsync(IHttpApplication application, CancellationToken cancellationToken)
where TContext : notnull
=> inner.StartAsync(application, cancellationToken);
public Task StopAsync(CancellationToken cancellationToken)
{
probe.MarkBegan();
return inner.StopAsync(cancellationToken);
}
public void Dispose() => inner.Dispose();
}
}
}