fix(localdb): end open sync streams on ApplicationStopping (lmxopcua#906)

LocalDbSyncService observed only the call token, which ASP.NET Core cancels
at the END of Kestrel's graceful drain. A peer's sync stream never ends on
its own, so every graceful stop of the passive node waited out the host's
whole ShutdownTimeout (30 s on OtOpcUa QA).

The service now takes an optional IHostApplicationLifetime (DI supplies it
in any generic host): ApplicationStopping is linked into the stream's token
and ends the session cleanly, and a stream arriving once stopping has begun
is refused Unavailable. Wire-compatible; no proto/schema change.

Tests: open stream ends within 5 s of StopApplication, new stream refused
Unavailable, control without a lifetime still running; both halves
mutation-killed. 262/262. Source only, NOT published (still 0.3.7).

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:29:31 -05:00
parent fa68b0ea02
commit 3ddb083b4c
2 changed files with 89 additions and 4 deletions
@@ -1,6 +1,7 @@
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using Grpc.Core; using Grpc.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using ZB.MOM.WW.LocalDb.Contracts; using ZB.MOM.WW.LocalDb.Contracts;
@@ -22,6 +23,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
private readonly ILogger<LocalDbSyncService> _logger; private readonly ILogger<LocalDbSyncService> _logger;
private readonly LocalDbMetrics? _metrics; private readonly LocalDbMetrics? _metrics;
private readonly SyncStatus? _status; private readonly SyncStatus? _status;
private readonly CancellationToken _stopping;
// The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service). // The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service).
private readonly object _gate = new(); private readonly object _gate = new();
@@ -48,10 +50,13 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
/// Creates the passive sync service. <paramref name="status"/> is taken as the public /// Creates the passive sync service. <paramref name="status"/> is taken as the public
/// <see cref="ISyncStatus"/> (SyncStatus is internal and cannot appear in a public signature); /// <see cref="ISyncStatus"/> (SyncStatus is internal and cannot appear in a public signature);
/// DI always supplies the concrete SyncStatus the ctor recovers by cast. /// DI always supplies the concrete SyncStatus the ctor recovers by cast.
/// <paramref name="lifetime"/>, when present (any generic host supplies it), ends every open
/// stream on <see cref="IHostApplicationLifetime.ApplicationStopping"/> — see <see cref="Sync"/>.
/// </summary> /// </summary>
public LocalDbSyncService( public LocalDbSyncService(
ILocalDb db, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory, ILocalDb db, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory,
LocalDbMetrics? metrics = null, ISyncStatus? status = null) LocalDbMetrics? metrics = null, ISyncStatus? status = null,
IHostApplicationLifetime? lifetime = null)
{ {
_db = SyncSessionFactory.RequireSqlite(db, nameof(LocalDbSyncService)); _db = SyncSessionFactory.RequireSqlite(db, nameof(LocalDbSyncService));
_options = options.Value; _options = options.Value;
@@ -59,6 +64,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
_logger = loggerFactory.CreateLogger<LocalDbSyncService>(); _logger = loggerFactory.CreateLogger<LocalDbSyncService>();
_metrics = metrics; _metrics = metrics;
_status = status as SyncStatus; _status = status as SyncStatus;
_stopping = lifetime?.ApplicationStopping ?? CancellationToken.None;
if (string.IsNullOrWhiteSpace(_options.ApiKey)) if (string.IsNullOrWhiteSpace(_options.ApiKey))
{ {
@@ -90,6 +96,14 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
/// that cannot be evicted within <see cref="TakeoverTimeout"/> is the one case where the old /// that cannot be evicted within <see cref="TakeoverTimeout"/> is the one case where the old
/// answer is still right — the newcomer refuses itself rather than double-running. /// answer is still right — the newcomer refuses itself rather than double-running.
/// </para> /// </para>
/// <para>
/// <b>Host shutdown ends the stream.</b> The call token fires only when the peer goes away or the
/// server aborts the request — and Kestrel aborts only at the END of its graceful drain, so through
/// 0.3.7 every graceful stop of the passive node waited out the host's whole <c>ShutdownTimeout</c>
/// (30 s by default) for a stream that never ends on its own (lmxopcua#906). The stream is now also
/// cancelled by <see cref="IHostApplicationLifetime.ApplicationStopping"/>, and a stream that
/// arrives once stopping has begun is refused <c>Unavailable</c> (the peer retries with backoff).
/// </para>
/// </remarks> /// </remarks>
public override async Task Sync( public override async Task Sync(
IAsyncStreamReader<SyncMessage> requestStream, IAsyncStreamReader<SyncMessage> requestStream,
@@ -108,7 +122,10 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
// path — including a session fault for a local (non-transport) reason, where the call token // path — including a session fault for a local (non-transport) reason, where the call token
// never fires and the pump would otherwise keep reading a stream whose handler has exited. // never fires and the pump would otherwise keep reading a stream whose handler has exited.
// It is also the handle a later call cancels to displace this one. // It is also the handle a later call cancels to displace this one.
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken); if (_stopping.IsCancellationRequested)
throw new RpcException(new Status(StatusCode.Unavailable, "this node is shutting down"));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, _stopping);
var mine = new Registration(linkedCts); var mine = new Registration(linkedCts);
try try
@@ -142,6 +159,11 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
{ {
// Client disconnect / server shutdown: the normal end of a long-lived streaming RPC. // Client disconnect / server shutdown: the normal end of a long-lived streaming RPC.
} }
catch (OperationCanceledException) when (_stopping.IsCancellationRequested)
{
// Host stopping: end the stream now rather than hold the server's graceful drain open.
_logger.LogInformation("Sync stream ended: host is stopping.");
}
catch (OperationCanceledException) catch (OperationCanceledException)
{ {
// Displaced by a newer stream: an orderly handover, not a failure of this node. // Displaced by a newer stream: an orderly handover, not a failure of this node.
@@ -1,5 +1,6 @@
using System.Threading.Channels; using System.Threading.Channels;
using Grpc.Core; using Grpc.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using ZB.MOM.WW.LocalDb.Contracts; using ZB.MOM.WW.LocalDb.Contracts;
@@ -35,7 +36,7 @@ public sealed class SyncStreamGuardTests : IDisposable
} }
} }
private LocalDbSyncService NewService() private LocalDbSyncService NewService(IHostApplicationLifetime? lifetime = null)
{ {
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db"); var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
_paths.Add(path); _paths.Add(path);
@@ -47,7 +48,8 @@ public sealed class SyncStreamGuardTests : IDisposable
return new LocalDbSyncService( return new LocalDbSyncService(
db, db,
Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }), Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }),
NullLoggerFactory.Instance); NullLoggerFactory.Instance,
lifetime: lifetime);
} }
private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct); private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct);
@@ -115,6 +117,67 @@ public sealed class SyncStreamGuardTests : IDisposable
await SwallowAsync(third); await SwallowAsync(third);
} }
[Fact]
public async Task HostStopping_EndsTheOpenStream_Cleanly()
{
// lmxopcua#906: the call token fires only at the END of Kestrel's graceful drain, so a stream
// that observed nothing else held every graceful stop of the passive node for the whole
// host ShutdownTimeout (30 s on QA). ApplicationStopping must end it now — and as an orderly
// end of the call, not a fault.
var lifetime = new FakeLifetime();
var service = NewService(lifetime);
using var callCts = new CancellationTokenSource(); // never cancelled: the peer is still there
var output = new RecordingWriter();
var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token));
await output.NextAsync(Timeout);
Assert.False(call.IsCompleted);
lifetime.StopApplication();
await call.WaitAsync(TimeSpan.FromSeconds(5)); // completes, and without throwing
}
[Fact]
public async Task HostStopping_RefusesANewStream_Unavailable()
{
// The peer redials the moment its stream ends; a node already stopping must not admit it.
var lifetime = new FakeLifetime();
var service = NewService(lifetime);
lifetime.StopApplication();
var ex = await Assert.ThrowsAsync<RpcException>(() =>
service.Sync(new SilentReader(), new RecordingWriter(), ContextFor(CancellationToken.None)));
Assert.Equal(StatusCode.Unavailable, ex.StatusCode);
}
[Fact]
public async Task WithoutALifetime_TheStreamRunsUntilItsCallEnds()
{
// Control for the two tests above: the same open stream, with no lifetime, is still running
// after the same wait — so their prompt end is the lifetime's doing, not the harness's.
var service = NewService();
using var callCts = new CancellationTokenSource();
var output = new RecordingWriter();
var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token));
await output.NextAsync(Timeout);
await Task.Delay(TimeSpan.FromMilliseconds(500));
Assert.False(call.IsCompleted);
callCts.Cancel();
await call.WaitAsync(Timeout);
}
private sealed class FakeLifetime : IHostApplicationLifetime
{
private readonly CancellationTokenSource _stopping = new();
public CancellationToken ApplicationStarted => CancellationToken.None;
public CancellationToken ApplicationStopping => _stopping.Token;
public CancellationToken ApplicationStopped => CancellationToken.None;
public void StopApplication() => _stopping.Cancel();
}
private static async Task SwallowAsync(Task task) private static async Task SwallowAsync(Task task)
{ {
try { await task.WaitAsync(Timeout); } try { await task.WaitAsync(Timeout); }