fix(localdb): end open sync streams on ApplicationStopping (lmxopcua#906)

LocalDbSyncService observed only the call token, which ASP.NET Core cancels
at the END of Kestrel's graceful drain. A peer's sync stream never ends on
its own, so every graceful stop of the passive node waited out the host's
whole ShutdownTimeout (30 s on OtOpcUa QA).

The service now takes an optional IHostApplicationLifetime (DI supplies it
in any generic host): ApplicationStopping is linked into the stream's token
and ends the session cleanly, and a stream arriving once stopping has begun
is refused Unavailable. Wire-compatible; no proto/schema change.

Tests: open stream ends within 5 s of StopApplication, new stream refused
Unavailable, control without a lifetime still running; both halves
mutation-killed. 262/262. Source only, NOT published (still 0.3.7).

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:29:31 -05:00
parent fa68b0ea02
commit 3ddb083b4c
2 changed files with 89 additions and 4 deletions
@@ -1,6 +1,7 @@
using System.Security.Cryptography;
using System.Text;
using Grpc.Core;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.LocalDb.Contracts;
@@ -22,6 +23,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
private readonly ILogger<LocalDbSyncService> _logger;
private readonly LocalDbMetrics? _metrics;
private readonly SyncStatus? _status;
private readonly CancellationToken _stopping;
// The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service).
private readonly object _gate = new();
@@ -48,10 +50,13 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
/// Creates the passive sync service. <paramref name="status"/> is taken as the public
/// <see cref="ISyncStatus"/> (SyncStatus is internal and cannot appear in a public signature);
/// DI always supplies the concrete SyncStatus the ctor recovers by cast.
/// <paramref name="lifetime"/>, when present (any generic host supplies it), ends every open
/// stream on <see cref="IHostApplicationLifetime.ApplicationStopping"/> — see <see cref="Sync"/>.
/// </summary>
public LocalDbSyncService(
ILocalDb db, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory,
LocalDbMetrics? metrics = null, ISyncStatus? status = null)
LocalDbMetrics? metrics = null, ISyncStatus? status = null,
IHostApplicationLifetime? lifetime = null)
{
_db = SyncSessionFactory.RequireSqlite(db, nameof(LocalDbSyncService));
_options = options.Value;
@@ -59,6 +64,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
_logger = loggerFactory.CreateLogger<LocalDbSyncService>();
_metrics = metrics;
_status = status as SyncStatus;
_stopping = lifetime?.ApplicationStopping ?? CancellationToken.None;
if (string.IsNullOrWhiteSpace(_options.ApiKey))
{
@@ -90,6 +96,14 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
/// that cannot be evicted within <see cref="TakeoverTimeout"/> is the one case where the old
/// answer is still right — the newcomer refuses itself rather than double-running.
/// </para>
/// <para>
/// <b>Host shutdown ends the stream.</b> The call token fires only when the peer goes away or the
/// server aborts the request — and Kestrel aborts only at the END of its graceful drain, so through
/// 0.3.7 every graceful stop of the passive node waited out the host's whole <c>ShutdownTimeout</c>
/// (30 s by default) for a stream that never ends on its own (lmxopcua#906). The stream is now also
/// cancelled by <see cref="IHostApplicationLifetime.ApplicationStopping"/>, and a stream that
/// arrives once stopping has begun is refused <c>Unavailable</c> (the peer retries with backoff).
/// </para>
/// </remarks>
public override async Task Sync(
IAsyncStreamReader<SyncMessage> requestStream,
@@ -108,7 +122,10 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
// path — including a session fault for a local (non-transport) reason, where the call token
// never fires and the pump would otherwise keep reading a stream whose handler has exited.
// It is also the handle a later call cancels to displace this one.
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
if (_stopping.IsCancellationRequested)
throw new RpcException(new Status(StatusCode.Unavailable, "this node is shutting down"));
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, _stopping);
var mine = new Registration(linkedCts);
try
@@ -142,6 +159,11 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
{
// Client disconnect / server shutdown: the normal end of a long-lived streaming RPC.
}
catch (OperationCanceledException) when (_stopping.IsCancellationRequested)
{
// Host stopping: end the stream now rather than hold the server's graceful drain open.
_logger.LogInformation("Sync stream ended: host is stopping.");
}
catch (OperationCanceledException)
{
// Displaced by a newer stream: an orderly handover, not a failure of this node.