fix(host): leave the cluster before Kestrel drains and end open streams on stop (#140)
A graceful stop of a site node whose peer holds an open LocalDb sync stream left the Akka cluster ~30 s late (the ScadaBridge half of lmxopcua#906, scadaproj#57). WebApplication stops the web server before AkkaHostedService, whose StopAsync ran CoordinatedShutdown, and Kestrel's graceful drain waited the whole ShutdownTimeout for /localdb_sync.v1.LocalDbSync/Sync: LocalDb 0.3.7 observes only the call token, which Kestrel cancels at the END of the drain. CancelAllStreams only covered SiteStreamGrpcServer's own streams. - AkkaHostedService is now an IHostedLifecycleService. StoppingAsync runs before any hosted service's StopAsync (Kestrel's included): marks the stop intentional, raises ApplicationStopping (so CancelAllStreams still precedes actor teardown), then runs and awaits CoordinatedShutdown(ClrExitReason), bounded by the host shutdown token. StopAsync returns the same completed shutdown. Exit codes unchanged: planned stop exits 0; only a termination outside host shutdown (self-down / #132 shut-out) exits 75. No Akka.Hosting here, so no ClusterLeavingReason/-1 exit-code path is involved. - ShutdownStreamCancellationInterceptor, innermost on both the central and the site AddGrpc pipelines: every streaming call's token is linked with ApplicationStopping; new streams are refused Unavailable while stopping. Covers the LocalDb sync stream until LocalDb 0.3.8 is pinned. Evidence (IntegrationTests/GracefulStopWithOpenSyncStreamTests: real WebApplication + Kestrel h2c + MapZbLocalDbSync behind LocalDbSyncAuthInterceptor + a real LocalDb SyncBackgroundService peer + the production AkkaHostedService forming a single-member cluster, 8 s drain budget, IServer wrapped to timestamp the drain): before 8859 ms stop, member removed at 8834 ms, drain began at 0 ms; after 960 ms stop, member removed at 950 ms, drain began at 959 ms. Mutations (all killed): StoppingAsync skips CoordinatedShutdown; interceptor passes the original context (stop 8003 ms); no Unavailable refusal; StoppingAsync does not raise ApplicationStopping; lifecycle interface dropped; site pipeline loses the interceptor. Release build of the solution: 0 warnings. Host.Tests 733/733, IntegrationTests 133/133, Communication.Tests 843/843. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,189 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#140 — open streaming gRPC calls end on <c>ApplicationStopping</c> instead of holding
|
||||
/// Kestrel's graceful drain for the whole shutdown timeout, and new streams are refused while stopping.
|
||||
/// The end-to-end measurement (real Kestrel, real LocalDb sync peer, real Akka node) lives in
|
||||
/// <c>IntegrationTests/GracefulStopWithOpenSyncStreamTests</c>.
|
||||
/// </summary>
|
||||
public sealed class ShutdownStreamCancellationInterceptorTests
|
||||
{
|
||||
private static (ShutdownStreamCancellationInterceptor Interceptor, CancellationTokenSource Stopping) Create()
|
||||
{
|
||||
var stopping = new CancellationTokenSource();
|
||||
var lifetime = Substitute.For<IHostApplicationLifetime>();
|
||||
lifetime.ApplicationStopping.Returns(stopping.Token);
|
||||
return (new ShutdownStreamCancellationInterceptor(
|
||||
lifetime, NullLogger<ShutdownStreamCancellationInterceptor>.Instance), stopping);
|
||||
}
|
||||
|
||||
/// <summary>An open server stream's token fires when the host starts stopping, and the call ends.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenServerStream_EndsOnApplicationStopping()
|
||||
{
|
||||
var (interceptor, stopping) = Create();
|
||||
using var callCts = new CancellationTokenSource();
|
||||
var context = new FakeServerCallContext("/svc/Stream", callCts.Token);
|
||||
var entered = new TaskCompletionSource();
|
||||
ServerCallContext? seen = null;
|
||||
|
||||
var call = interceptor.ServerStreamingServerHandler<string, string>(
|
||||
"req", null!, context, async (_, _, ctx) =>
|
||||
{
|
||||
seen = ctx;
|
||||
entered.SetResult();
|
||||
await Task.Delay(Timeout.Infinite, ctx.CancellationToken).ContinueWith(_ => { });
|
||||
});
|
||||
|
||||
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
Assert.False(call.IsCompleted);
|
||||
Assert.Equal("/svc/Stream", seen!.Method);
|
||||
|
||||
stopping.Cancel();
|
||||
|
||||
await call.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
Assert.False(callCts.IsCancellationRequested, "the call's own token is not what ended it");
|
||||
}
|
||||
|
||||
/// <summary>The handler still ends when the client goes away (the call token), as before.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenDuplexStream_StillEndsOnTheCallToken()
|
||||
{
|
||||
var (interceptor, _) = Create();
|
||||
using var callCts = new CancellationTokenSource();
|
||||
var context = new FakeServerCallContext("/svc/Duplex", callCts.Token);
|
||||
|
||||
var call = interceptor.DuplexStreamingServerHandler<string, string>(
|
||||
null!, null!, context,
|
||||
(_, _, ctx) => Task.Delay(Timeout.Infinite, ctx.CancellationToken).ContinueWith(_ => { }));
|
||||
|
||||
callCts.Cancel();
|
||||
await call.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>A client stream returns its handler's response through the wrapper.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ClientStream_ReturnsTheHandlerResponse()
|
||||
{
|
||||
var (interceptor, _) = Create();
|
||||
var context = new FakeServerCallContext("/svc/Upload", CancellationToken.None);
|
||||
|
||||
var response = await interceptor.ClientStreamingServerHandler<string, string>(
|
||||
null!, context, (_, _) => Task.FromResult("done"));
|
||||
|
||||
Assert.Equal("done", response);
|
||||
}
|
||||
|
||||
/// <summary>Once stopping has begun, a new stream is refused with the retryable Unavailable.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task NewStream_WhileStopping_IsRefusedUnavailable()
|
||||
{
|
||||
var (interceptor, stopping) = Create();
|
||||
stopping.Cancel();
|
||||
var invoked = false;
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() => interceptor.DuplexStreamingServerHandler<string, string>(
|
||||
null!, null!, new FakeServerCallContext("/localdb_sync.v1.LocalDbSync/Sync", CancellationToken.None),
|
||||
(_, _, _) => { invoked = true; return Task.CompletedTask; }));
|
||||
|
||||
Assert.Equal(StatusCode.Unavailable, ex.StatusCode);
|
||||
Assert.False(invoked);
|
||||
}
|
||||
|
||||
/// <summary>Unary calls are left alone even while stopping — finishing them is what the drain is for.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task UnaryCall_WhileStopping_PassesThrough()
|
||||
{
|
||||
var (interceptor, stopping) = Create();
|
||||
stopping.Cancel();
|
||||
|
||||
var reply = await interceptor.UnaryServerHandler<string, string>(
|
||||
"req", new FakeServerCallContext("/svc/Unary", CancellationToken.None), (_, _) => Task.FromResult("ok"));
|
||||
|
||||
Assert.Equal("ok", reply);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Program.cs registers the interceptor LAST (innermost, after the auth gates) on BOTH the
|
||||
/// central and the site <c>AddGrpc</c> pipelines.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Program_RegistersTheInterceptorInnermost_OnBothGrpcPipelines()
|
||||
{
|
||||
var program = Path.Combine(FindRepositoryRoot(), "src", "ZB.MOM.WW.ScadaBridge.Host", "Program.cs");
|
||||
var root = CSharpSyntaxTree.ParseText(File.ReadAllText(program)).GetRoot();
|
||||
|
||||
var addGrpcLambdas = root.DescendantNodes().OfType<InvocationExpressionSyntax>()
|
||||
.Where(i => i.Expression is MemberAccessExpressionSyntax { Name.Identifier.ValueText: "AddGrpc" })
|
||||
.Select(i => i.ArgumentList.Arguments.Single().Expression)
|
||||
.OfType<LambdaExpressionSyntax>()
|
||||
.ToList();
|
||||
Assert.Equal(2, addGrpcLambdas.Count);
|
||||
|
||||
foreach (var lambda in addGrpcLambdas)
|
||||
{
|
||||
var added = lambda.DescendantNodes().OfType<GenericNameSyntax>()
|
||||
.Where(n => n.Identifier.ValueText == "Add" && n.Parent is MemberAccessExpressionSyntax m
|
||||
&& m.Expression.ToString().EndsWith("Interceptors", StringComparison.Ordinal))
|
||||
.Select(n => n.TypeArgumentList.Arguments.Single().ToString())
|
||||
.ToList();
|
||||
Assert.True(added.Count >= 2, $"expected an auth gate and the shutdown interceptor, got [{string.Join(", ", added)}]");
|
||||
Assert.Equal(nameof(ShutdownStreamCancellationInterceptor), added[^1]);
|
||||
}
|
||||
}
|
||||
|
||||
private static string FindRepositoryRoot()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.ScadaBridge.slnx")))
|
||||
dir = dir.Parent;
|
||||
return dir?.FullName ?? throw new InvalidOperationException("repository root not found");
|
||||
}
|
||||
|
||||
/// <summary>Minimal hand-rolled <see cref="ServerCallContext"/> (grpc-dotnet ships no test context).</summary>
|
||||
private sealed class FakeServerCallContext(string method, CancellationToken token) : ServerCallContext
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override string MethodCore => method;
|
||||
/// <inheritdoc />
|
||||
protected override string HostCore => "localhost";
|
||||
/// <inheritdoc />
|
||||
protected override string PeerCore => "ipv4:127.0.0.1:12345";
|
||||
/// <inheritdoc />
|
||||
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
|
||||
/// <inheritdoc />
|
||||
protected override Metadata RequestHeadersCore { get; } = [];
|
||||
/// <inheritdoc />
|
||||
protected override CancellationToken CancellationTokenCore => token;
|
||||
/// <inheritdoc />
|
||||
protected override Metadata ResponseTrailersCore { get; } = [];
|
||||
/// <inheritdoc />
|
||||
protected override Status StatusCore { get; set; }
|
||||
/// <inheritdoc />
|
||||
protected override WriteOptions? WriteOptionsCore { get; set; }
|
||||
/// <inheritdoc />
|
||||
protected override AuthContext AuthContextCore { get; } =
|
||||
new(null, new Dictionary<string, List<AuthProperty>>());
|
||||
/// <inheritdoc />
|
||||
protected override IDictionary<object, object> UserStateCore { get; } = new Dictionary<object, object>();
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,54 @@ public sealed class UnexpectedTerminationTests
|
||||
Assert.False(service.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#140: the lifecycle hook that runs before Kestrel drains raises ApplicationStopping
|
||||
/// FIRST (so the site's stream cancellation still precedes actor teardown), then completes the
|
||||
/// CoordinatedShutdown — and a planned stop is never mistaken for a self-down (no exit code 75).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StoppingAsync_RaisesStopping_ThenShutsDown_AndIsNotASelfDown()
|
||||
{
|
||||
var lifetime = Substitute.For<IHostApplicationLifetime>();
|
||||
var service = CreateAkkaHostedService(lifetime);
|
||||
var system = service.GetOrCreateActorSystem();
|
||||
bool? terminatedWhenStoppingRaised = null;
|
||||
lifetime.When(l => l.StopApplication()).Do(_ => terminatedWhenStoppingRaised = system.WhenTerminated.IsCompleted);
|
||||
|
||||
await service.StoppingAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(system.WhenTerminated.IsCompleted, "StoppingAsync must complete the CoordinatedShutdown");
|
||||
Assert.Equal(false, terminatedWhenStoppingRaised);
|
||||
Assert.Equal(CoordinatedShutdown.ClrExitReason.Instance, CoordinatedShutdown.Get(system).ShutdownReason);
|
||||
|
||||
// The host's later StopAsync is the same, completed shutdown — not a second one.
|
||||
await service.StopAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(5));
|
||||
await Task.Delay(500);
|
||||
|
||||
lifetime.Received(1).StopApplication();
|
||||
Assert.False(service.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
/// <summary>A host whose Akka service never created its system: StoppingAsync must not create one.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StoppingAsync_WithoutAnActorSystem_DoesNotCreateOne()
|
||||
{
|
||||
var lifetime = Substitute.For<IHostApplicationLifetime>();
|
||||
var service = CreateAkkaHostedService(lifetime);
|
||||
|
||||
await service.StoppingAsync(CancellationToken.None);
|
||||
|
||||
Assert.Null(service.ActorSystem);
|
||||
lifetime.Received(1).StopApplication();
|
||||
}
|
||||
|
||||
/// <summary>ScadaBridge#140: the host only calls StoppingAsync on a service that declares the lifecycle interface.</summary>
|
||||
[Fact]
|
||||
public void AkkaHostedService_IsAHostedLifecycleService()
|
||||
=> Assert.True(typeof(IHostedLifecycleService).IsAssignableFrom(typeof(AkkaHostedService)));
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#132: the self-down exit code is non-zero (Windows service recovery with
|
||||
/// failureflag 1 restarts only on a non-zero exit) and is only pushed into a Windows service
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
using System.Diagnostics;
|
||||
using System.Net;
|
||||
using Akka.Actor;
|
||||
using Akka.Cluster;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Xunit.Abstractions;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.LocalDb.Replication;
|
||||
using ZB.MOM.WW.ScadaBridge.ClusterInfrastructure;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.ScadaBridge.Host;
|
||||
using ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
using AkkaCluster = Akka.Cluster.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.IntegrationTests;
|
||||
|
||||
/// <summary>
|
||||
/// ScadaBridge#140 (the ScadaBridge half of lmxopcua#906): a graceful stop of a site node whose
|
||||
/// peer holds an open LocalDb sync stream must leave the Akka cluster FIRST and finish promptly,
|
||||
/// not wait out Kestrel's graceful-drain timeout.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The stopping node is a real <see cref="WebApplication"/> — the hosting model whose hosted-service
|
||||
/// order caused the bug (the web server stops before the Akka service) — with a real Kestrel h2c
|
||||
/// listener, the real <c>MapZbLocalDbSync</c> behind the real <see cref="LocalDbSyncAuthInterceptor"/>,
|
||||
/// the production <see cref="AkkaHostedService"/> forming a real single-member Akka cluster, and the
|
||||
/// production <see cref="ShutdownStreamCancellationInterceptor"/>. The peer is a real LocalDb
|
||||
/// <c>SyncBackgroundService</c> dialling it over loopback and holding its stream open.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The web server's <see cref="IServer"/> is wrapped to timestamp the moment its <c>StopAsync</c> (the
|
||||
/// drain) begins, so "left the cluster before the drain" is asserted as an ORDER, not inferred from
|
||||
/// a total duration.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The second test is the positive control: the same stack as it was before the fix — the Akka service
|
||||
/// seen by the host as a plain <see cref="IHostedService"/> (so its shutdown runs only in
|
||||
/// <c>StopAsync</c>) and no stream interceptor — reproduces the bug: the stop waits out the whole
|
||||
/// shutdown timeout and the cluster is left only after the drain. Without it, a fast stop in the first
|
||||
/// test could be a harness that never held a stream open at all.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
[Collection("LocalDbSitePairConvergence")]
|
||||
public sealed class GracefulStopWithOpenSyncStreamTests
|
||||
{
|
||||
private const string ApiKey = "scadabridge-140-graceful-stop-key";
|
||||
|
||||
/// <summary>The host's ShutdownTimeout in these tests — Kestrel's drain budget (30 s in production).</summary>
|
||||
private static readonly TimeSpan ShutdownTimeout = TimeSpan.FromSeconds(8);
|
||||
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
static GracefulStopWithOpenSyncStreamTests() =>
|
||||
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
|
||||
|
||||
/// <summary>Creates the test class.</summary>
|
||||
/// <param name="output">xunit output, for the measured timings.</param>
|
||||
public GracefulStopWithOpenSyncStreamTests(ITestOutputHelper output) => _output = output;
|
||||
|
||||
/// <summary>With the fix: the node leaves the cluster before the drain begins and stops well inside the budget.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task GracefulStop_WithOpenSyncStream_LeavesClusterBeforeDrain_AndFinishesPromptly()
|
||||
{
|
||||
var result = await RunStopScenarioAsync(withFix: true);
|
||||
|
||||
// The peer's stream was open and was ended by the stop (not by a peer that never connected).
|
||||
Assert.True(result.StreamWasOpen, "the peer's sync stream was never open on the stopping node");
|
||||
|
||||
// Left the cluster, and did so before the web server began draining.
|
||||
Assert.NotNull(result.MemberRemovedAt);
|
||||
Assert.NotNull(result.ServerStopBeganAt);
|
||||
Assert.True(result.MemberRemovedAt < result.ServerStopBeganAt,
|
||||
$"the node must leave the cluster BEFORE Kestrel drains (removed at {result.MemberRemovedAt?.TotalMilliseconds:0} ms, " +
|
||||
$"drain began at {result.ServerStopBeganAt?.TotalMilliseconds:0} ms)");
|
||||
|
||||
// Finished well inside the drain budget: the open sync stream did not hold the stop.
|
||||
Assert.True(result.StopElapsed < TimeSpan.FromSeconds(4),
|
||||
$"stop took {result.StopElapsed.TotalMilliseconds:0} ms against a {ShutdownTimeout.TotalSeconds:0} s drain budget");
|
||||
|
||||
// A planned stop is not a self-down: no exit code 75.
|
||||
Assert.False(result.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
/// <summary>Positive control: the pre-fix wiring reproduces the late leave and the full-budget stop.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task PositiveControl_WithoutTheFix_StopWaitsOutTheDrain_AndLeavesTheClusterLast()
|
||||
{
|
||||
var result = await RunStopScenarioAsync(withFix: false);
|
||||
|
||||
Assert.True(result.StreamWasOpen);
|
||||
Assert.True(result.StopElapsed >= ShutdownTimeout - TimeSpan.FromSeconds(1),
|
||||
$"stop took {result.StopElapsed.TotalMilliseconds:0} ms; expected it to wait out the {ShutdownTimeout.TotalSeconds:0} s drain");
|
||||
Assert.NotNull(result.MemberRemovedAt);
|
||||
Assert.True(result.MemberRemovedAt > result.ServerStopBeganAt);
|
||||
Assert.True(result.MemberRemovedAt >= ShutdownTimeout - TimeSpan.FromSeconds(1));
|
||||
Assert.False(result.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
|
||||
private sealed record StopResult(
|
||||
TimeSpan StopElapsed,
|
||||
TimeSpan? MemberRemovedAt,
|
||||
TimeSpan? ServerStopBeganAt,
|
||||
bool StreamWasOpen,
|
||||
bool TerminatedOutsideHostShutdown);
|
||||
|
||||
private async Task<StopResult> RunStopScenarioAsync(bool withFix)
|
||||
{
|
||||
var pathA = Path.Combine(Path.GetTempPath(), $"sb140-A-{Guid.NewGuid():N}.db");
|
||||
var pathB = Path.Combine(Path.GetTempPath(), $"sb140-B-{Guid.NewGuid():N}.db");
|
||||
await using var dbA = BuildDatabaseProvider(pathA, "node-a");
|
||||
await using var dbB = BuildDatabaseProvider(pathB, "node-b");
|
||||
var clock = new Stopwatch();
|
||||
var probe = new ServerStopProbe(clock);
|
||||
IHost? initiator = null;
|
||||
WebApplication? passive = null;
|
||||
|
||||
try
|
||||
{
|
||||
passive = BuildStoppingNode(dbB.GetRequiredService<ILocalDb>(), probe, withFix);
|
||||
await passive.StartAsync();
|
||||
|
||||
var akka = passive.Services.GetRequiredService<AkkaHostedService>();
|
||||
var cluster = AkkaCluster.Get(akka.ActorSystem!);
|
||||
await WaitUntilAsync(() => cluster.State.Members.Any(m => m.Status == MemberStatus.Up),
|
||||
"the node to come Up as its own seed");
|
||||
|
||||
TimeSpan? memberRemovedAt = null;
|
||||
cluster.RegisterOnMemberRemoved(() => memberRemovedAt ??= clock.Elapsed);
|
||||
|
||||
var address = passive.Services.GetRequiredService<IServer>()
|
||||
.Features.Get<IServerAddressesFeature>()!.Addresses.Single();
|
||||
initiator = await StartInitiatorAsync(dbA.GetRequiredService<ILocalDb>(), address);
|
||||
|
||||
// The peer's stream is open and holding a session on the stopping node.
|
||||
var status = passive.Services.GetRequiredService<ISyncStatus>();
|
||||
await WaitUntilAsync(() => status.Connected, "the peer's sync stream to be open on the stopping node");
|
||||
var streamWasOpen = status.Connected;
|
||||
|
||||
clock.Start();
|
||||
await passive.StopAsync(CancellationToken.None);
|
||||
var elapsed = clock.Elapsed;
|
||||
|
||||
_output.WriteLine(
|
||||
$"withFix={withFix}: stop {elapsed.TotalMilliseconds:0} ms; member removed at " +
|
||||
$"{memberRemovedAt?.TotalMilliseconds:0} ms; web-server drain began at " +
|
||||
$"{probe.BeganAt?.TotalMilliseconds:0} ms (drain budget {ShutdownTimeout.TotalSeconds:0} s)");
|
||||
return new StopResult(elapsed, memberRemovedAt, probe.BeganAt, streamWasOpen,
|
||||
akka.TerminatedOutsideHostShutdown);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (initiator is not null)
|
||||
{
|
||||
try { await initiator.StopAsync(TimeSpan.FromSeconds(5)); } catch { /* teardown */ }
|
||||
initiator.Dispose();
|
||||
}
|
||||
|
||||
if (passive is not null)
|
||||
await passive.DisposeAsync();
|
||||
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (var path in new[] { pathA, pathB })
|
||||
foreach (var suffix in new[] { "", "-wal", "-shm" })
|
||||
try { File.Delete(path + suffix); } catch { /* best effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
private static WebApplication BuildStoppingNode(ILocalDb db, ServerStopProbe probe, bool withFix)
|
||||
{
|
||||
var builder = WebApplication.CreateBuilder();
|
||||
builder.Environment.ApplicationName = typeof(GracefulStopWithOpenSyncStreamTests).Assembly.GetName().Name!;
|
||||
builder.Logging.ClearProviders();
|
||||
builder.WebHost.ConfigureKestrel(k =>
|
||||
k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
|
||||
builder.Services.Configure<HostOptions>(o => o.ShutdownTimeout = ShutdownTimeout);
|
||||
|
||||
// The production site interceptor chain: the real auth gate, then (with the fix) the
|
||||
// stream-cancellation interceptor innermost, exactly as Program.cs orders them.
|
||||
builder.Services.AddGrpc(o =>
|
||||
{
|
||||
o.Interceptors.Add<LocalDbSyncAuthInterceptor>();
|
||||
if (withFix)
|
||||
o.Interceptors.Add<ShutdownStreamCancellationInterceptor>();
|
||||
});
|
||||
builder.Services.AddSingleton(db);
|
||||
builder.Services.AddZbLocalDbReplication(ReplicationConfig(peerAddress: null));
|
||||
|
||||
// The production AkkaHostedService, registered the way Program/SiteServiceRegistration do.
|
||||
// Role "StopProbe" (neither Central nor Site) makes StartAsync register only the dead-letter
|
||||
// monitor; the cluster formation, CoordinatedShutdown and the WhenTerminated watchdog are
|
||||
// the production code paths.
|
||||
var port = FreePort();
|
||||
builder.Services.AddSingleton(sp => new AkkaHostedService(
|
||||
sp,
|
||||
Options.Create(new NodeOptions { Role = "StopProbe", NodeHostname = "127.0.0.1", RemotingPort = port }),
|
||||
Options.Create(new ClusterOptions
|
||||
{
|
||||
SeedNodes = new List<string> { $"akka.tcp://scadabridge@127.0.0.1:{port}" },
|
||||
AllowSingleNodeCluster = true,
|
||||
MinNrOfMembers = 1,
|
||||
}),
|
||||
Options.Create(new CommunicationOptions()),
|
||||
sp.GetRequiredService<ILogger<AkkaHostedService>>(),
|
||||
sp.GetRequiredService<IHostApplicationLifetime>(),
|
||||
sp.GetRequiredService<IHostLifetime>()));
|
||||
if (withFix)
|
||||
builder.Services.AddHostedService(sp => sp.GetRequiredService<AkkaHostedService>());
|
||||
else
|
||||
builder.Services.AddHostedService(sp => new PreFixHostedService(sp.GetRequiredService<AkkaHostedService>()));
|
||||
|
||||
// Wrap the web server so the start of its drain is timestamped.
|
||||
var serverDescriptor = builder.Services.Last(d => d.ServiceType == typeof(IServer));
|
||||
builder.Services.Remove(serverDescriptor);
|
||||
builder.Services.AddSingleton<IServer>(sp =>
|
||||
{
|
||||
var inner = serverDescriptor.ImplementationFactory is { } factory
|
||||
? (IServer)factory(sp)
|
||||
: (IServer)ActivatorUtilities.CreateInstance(sp, serverDescriptor.ImplementationType!);
|
||||
return probe.Wrap(inner);
|
||||
});
|
||||
|
||||
var app = builder.Build();
|
||||
app.MapZbLocalDbSync();
|
||||
return app;
|
||||
}
|
||||
|
||||
private static async Task<IHost> StartInitiatorAsync(ILocalDb db, string peerAddress)
|
||||
{
|
||||
var config = ReplicationConfig(peerAddress);
|
||||
var host = new HostBuilder()
|
||||
.ConfigureServices(services =>
|
||||
{
|
||||
services.AddLogging();
|
||||
services.AddSingleton(db);
|
||||
services.AddZbLocalDbReplication(config);
|
||||
})
|
||||
.Build();
|
||||
await host.StartAsync();
|
||||
return host;
|
||||
}
|
||||
|
||||
private static IConfiguration ReplicationConfig(string? peerAddress)
|
||||
{
|
||||
var values = new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Replication:FlushInterval"] = "00:00:00.050",
|
||||
["LocalDb:Replication:ReconnectBackoffMax"] = "00:00:02",
|
||||
["LocalDb:Replication:ApiKey"] = ApiKey,
|
||||
};
|
||||
if (peerAddress is not null)
|
||||
values["LocalDb:Replication:PeerAddress"] = peerAddress;
|
||||
return new ConfigurationBuilder().AddInMemoryCollection(values).Build();
|
||||
}
|
||||
|
||||
private static ServiceProvider BuildDatabaseProvider(string path, string nodeName)
|
||||
{
|
||||
string Absent() => Path.Combine(Path.GetTempPath(), $"absent-{Guid.NewGuid():N}.db");
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["LocalDb:Path"] = path,
|
||||
["ScadaBridge:Node:NodeName"] = nodeName,
|
||||
["LocalDb:Replication:ApiKey"] = ApiKey,
|
||||
// Legacy migrators pointed at paths that do not exist, so they no-op.
|
||||
["ScadaBridge:OperationTracking:ConnectionString"] = $"Data Source={Absent()}",
|
||||
["ScadaBridge:SiteEventLog:DatabasePath"] = Absent(),
|
||||
["ScadaBridge:StoreAndForward:SqliteDbPath"] = Absent(),
|
||||
["ScadaBridge:Database:SiteDbPath"] = Absent(),
|
||||
})
|
||||
.Build();
|
||||
|
||||
return new ServiceCollection()
|
||||
.AddZbLocalDb(config, db => SiteLocalDbSetup.OnReady(db, config))
|
||||
.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static int FreePort()
|
||||
{
|
||||
var l = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
|
||||
l.Start();
|
||||
var p = ((IPEndPoint)l.LocalEndpoint).Port;
|
||||
l.Stop();
|
||||
return p;
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
if (condition())
|
||||
return;
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
Assert.Fail($"Timed out waiting for: {because}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The pre-fix registration shape: the host sees the Akka service only as an
|
||||
/// <see cref="IHostedService"/>, so its shutdown runs in <c>StopAsync</c>, after Kestrel's.
|
||||
/// </summary>
|
||||
private sealed class PreFixHostedService(AkkaHostedService inner) : IHostedService
|
||||
{
|
||||
public Task StartAsync(CancellationToken cancellationToken) => inner.StartAsync(cancellationToken);
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => inner.StopAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Records when the wrapped web server's <c>StopAsync</c> (its graceful drain) begins.</summary>
|
||||
private sealed class ServerStopProbe(Stopwatch clock)
|
||||
{
|
||||
public TimeSpan? BeganAt { get; private set; }
|
||||
|
||||
private void MarkBegan() => BeganAt ??= clock.Elapsed;
|
||||
|
||||
public IServer Wrap(IServer inner) => new RecordingServer(inner, this);
|
||||
|
||||
private sealed class RecordingServer(IServer inner, ServerStopProbe probe) : IServer
|
||||
{
|
||||
public IFeatureCollection Features => inner.Features;
|
||||
|
||||
public Task StartAsync<TContext>(IHttpApplication<TContext> application, CancellationToken cancellationToken)
|
||||
where TContext : notnull
|
||||
=> inner.StartAsync(application, cancellationToken);
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
probe.MarkBegan();
|
||||
return inner.StopAsync(cancellationToken);
|
||||
}
|
||||
|
||||
public void Dispose() => inner.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user