Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.Host.Tests/ShutdownStreamCancellationInterceptorTests.cs
T
Joseph Doherty 1a6108c402 fix(host): leave the cluster before Kestrel drains and end open streams on stop (#140)
A graceful stop of a site node whose peer holds an open LocalDb sync stream
left the Akka cluster ~30 s late (the ScadaBridge half of lmxopcua#906,
scadaproj#57). WebApplication stops the web server before AkkaHostedService,
whose StopAsync ran CoordinatedShutdown, and Kestrel's graceful drain waited
the whole ShutdownTimeout for /localdb_sync.v1.LocalDbSync/Sync: LocalDb 0.3.7
observes only the call token, which Kestrel cancels at the END of the drain.
CancelAllStreams only covered SiteStreamGrpcServer's own streams.

- AkkaHostedService is now an IHostedLifecycleService. StoppingAsync runs
  before any hosted service's StopAsync (Kestrel's included): marks the stop
  intentional, raises ApplicationStopping (so CancelAllStreams still precedes
  actor teardown), then runs and awaits CoordinatedShutdown(ClrExitReason),
  bounded by the host shutdown token. StopAsync returns the same completed
  shutdown. Exit codes unchanged: planned stop exits 0; only a termination
  outside host shutdown (self-down / #132 shut-out) exits 75. No Akka.Hosting
  here, so no ClusterLeavingReason/-1 exit-code path is involved.
- ShutdownStreamCancellationInterceptor, innermost on both the central and the
  site AddGrpc pipelines: every streaming call's token is linked with
  ApplicationStopping; new streams are refused Unavailable while stopping.
  Covers the LocalDb sync stream until LocalDb 0.3.8 is pinned.

Evidence (IntegrationTests/GracefulStopWithOpenSyncStreamTests: real
WebApplication + Kestrel h2c + MapZbLocalDbSync behind LocalDbSyncAuthInterceptor
+ a real LocalDb SyncBackgroundService peer + the production AkkaHostedService
forming a single-member cluster, 8 s drain budget, IServer wrapped to timestamp
the drain): before 8859 ms stop, member removed at 8834 ms, drain began at 0 ms;
after 960 ms stop, member removed at 950 ms, drain began at 959 ms.

Mutations (all killed): StoppingAsync skips CoordinatedShutdown; interceptor
passes the original context (stop 8003 ms); no Unavailable refusal; StoppingAsync
does not raise ApplicationStopping; lifecycle interface dropped; site pipeline
loses the interceptor.

Release build of the solution: 0 warnings. Host.Tests 733/733,
IntegrationTests 133/133, Communication.Tests 843/843.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 04:16:42 -05:00

190 lines
8.5 KiB
C#

using Grpc.Core;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
namespace ZB.MOM.WW.ScadaBridge.Host.Tests;
/// <summary>
/// ScadaBridge#140 — open streaming gRPC calls end on <c>ApplicationStopping</c> instead of holding
/// Kestrel's graceful drain for the whole shutdown timeout, and new streams are refused while stopping.
/// The end-to-end measurement (real Kestrel, real LocalDb sync peer, real Akka node) lives in
/// <c>IntegrationTests/GracefulStopWithOpenSyncStreamTests</c>.
/// </summary>
public sealed class ShutdownStreamCancellationInterceptorTests
{
private static (ShutdownStreamCancellationInterceptor Interceptor, CancellationTokenSource Stopping) Create()
{
var stopping = new CancellationTokenSource();
var lifetime = Substitute.For<IHostApplicationLifetime>();
lifetime.ApplicationStopping.Returns(stopping.Token);
return (new ShutdownStreamCancellationInterceptor(
lifetime, NullLogger<ShutdownStreamCancellationInterceptor>.Instance), stopping);
}
/// <summary>An open server stream's token fires when the host starts stopping, and the call ends.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenServerStream_EndsOnApplicationStopping()
{
var (interceptor, stopping) = Create();
using var callCts = new CancellationTokenSource();
var context = new FakeServerCallContext("/svc/Stream", callCts.Token);
var entered = new TaskCompletionSource();
ServerCallContext? seen = null;
var call = interceptor.ServerStreamingServerHandler<string, string>(
"req", null!, context, async (_, _, ctx) =>
{
seen = ctx;
entered.SetResult();
await Task.Delay(Timeout.Infinite, ctx.CancellationToken).ContinueWith(_ => { });
});
await entered.Task.WaitAsync(TimeSpan.FromSeconds(5));
Assert.False(call.IsCompleted);
Assert.Equal("/svc/Stream", seen!.Method);
stopping.Cancel();
await call.WaitAsync(TimeSpan.FromSeconds(5));
Assert.False(callCts.IsCancellationRequested, "the call's own token is not what ended it");
}
/// <summary>The handler still ends when the client goes away (the call token), as before.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task OpenDuplexStream_StillEndsOnTheCallToken()
{
var (interceptor, _) = Create();
using var callCts = new CancellationTokenSource();
var context = new FakeServerCallContext("/svc/Duplex", callCts.Token);
var call = interceptor.DuplexStreamingServerHandler<string, string>(
null!, null!, context,
(_, _, ctx) => Task.Delay(Timeout.Infinite, ctx.CancellationToken).ContinueWith(_ => { }));
callCts.Cancel();
await call.WaitAsync(TimeSpan.FromSeconds(5));
}
/// <summary>A client stream returns its handler's response through the wrapper.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task ClientStream_ReturnsTheHandlerResponse()
{
var (interceptor, _) = Create();
var context = new FakeServerCallContext("/svc/Upload", CancellationToken.None);
var response = await interceptor.ClientStreamingServerHandler<string, string>(
null!, context, (_, _) => Task.FromResult("done"));
Assert.Equal("done", response);
}
/// <summary>Once stopping has begun, a new stream is refused with the retryable Unavailable.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task NewStream_WhileStopping_IsRefusedUnavailable()
{
var (interceptor, stopping) = Create();
stopping.Cancel();
var invoked = false;
var ex = await Assert.ThrowsAsync<RpcException>(() => interceptor.DuplexStreamingServerHandler<string, string>(
null!, null!, new FakeServerCallContext("/localdb_sync.v1.LocalDbSync/Sync", CancellationToken.None),
(_, _, _) => { invoked = true; return Task.CompletedTask; }));
Assert.Equal(StatusCode.Unavailable, ex.StatusCode);
Assert.False(invoked);
}
/// <summary>Unary calls are left alone even while stopping — finishing them is what the drain is for.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task UnaryCall_WhileStopping_PassesThrough()
{
var (interceptor, stopping) = Create();
stopping.Cancel();
var reply = await interceptor.UnaryServerHandler<string, string>(
"req", new FakeServerCallContext("/svc/Unary", CancellationToken.None), (_, _) => Task.FromResult("ok"));
Assert.Equal("ok", reply);
}
/// <summary>
/// Program.cs registers the interceptor LAST (innermost, after the auth gates) on BOTH the
/// central and the site <c>AddGrpc</c> pipelines.
/// </summary>
[Fact]
public void Program_RegistersTheInterceptorInnermost_OnBothGrpcPipelines()
{
var program = Path.Combine(FindRepositoryRoot(), "src", "ZB.MOM.WW.ScadaBridge.Host", "Program.cs");
var root = CSharpSyntaxTree.ParseText(File.ReadAllText(program)).GetRoot();
var addGrpcLambdas = root.DescendantNodes().OfType<InvocationExpressionSyntax>()
.Where(i => i.Expression is MemberAccessExpressionSyntax { Name.Identifier.ValueText: "AddGrpc" })
.Select(i => i.ArgumentList.Arguments.Single().Expression)
.OfType<LambdaExpressionSyntax>()
.ToList();
Assert.Equal(2, addGrpcLambdas.Count);
foreach (var lambda in addGrpcLambdas)
{
var added = lambda.DescendantNodes().OfType<GenericNameSyntax>()
.Where(n => n.Identifier.ValueText == "Add" && n.Parent is MemberAccessExpressionSyntax m
&& m.Expression.ToString().EndsWith("Interceptors", StringComparison.Ordinal))
.Select(n => n.TypeArgumentList.Arguments.Single().ToString())
.ToList();
Assert.True(added.Count >= 2, $"expected an auth gate and the shutdown interceptor, got [{string.Join(", ", added)}]");
Assert.Equal(nameof(ShutdownStreamCancellationInterceptor), added[^1]);
}
}
private static string FindRepositoryRoot()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "ZB.MOM.WW.ScadaBridge.slnx")))
dir = dir.Parent;
return dir?.FullName ?? throw new InvalidOperationException("repository root not found");
}
/// <summary>Minimal hand-rolled <see cref="ServerCallContext"/> (grpc-dotnet ships no test context).</summary>
private sealed class FakeServerCallContext(string method, CancellationToken token) : ServerCallContext
{
/// <inheritdoc />
protected override string MethodCore => method;
/// <inheritdoc />
protected override string HostCore => "localhost";
/// <inheritdoc />
protected override string PeerCore => "ipv4:127.0.0.1:12345";
/// <inheritdoc />
protected override DateTime DeadlineCore => DateTime.UtcNow.AddMinutes(1);
/// <inheritdoc />
protected override Metadata RequestHeadersCore { get; } = [];
/// <inheritdoc />
protected override CancellationToken CancellationTokenCore => token;
/// <inheritdoc />
protected override Metadata ResponseTrailersCore { get; } = [];
/// <inheritdoc />
protected override Status StatusCore { get; set; }
/// <inheritdoc />
protected override WriteOptions? WriteOptionsCore { get; set; }
/// <inheritdoc />
protected override AuthContext AuthContextCore { get; } =
new(null, new Dictionary<string, List<AuthProperty>>());
/// <inheritdoc />
protected override IDictionary<object, object> UserStateCore { get; } = new Dictionary<object, object>();
/// <inheritdoc />
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options)
=> throw new NotSupportedException();
/// <inheritdoc />
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
}
}