fix(localdb): end open sync streams on ApplicationStopping (lmxopcua#906)
LocalDbSyncService observed only the call token, which ASP.NET Core cancels at the END of Kestrel's graceful drain. A peer's sync stream never ends on its own, so every graceful stop of the passive node waited out the host's whole ShutdownTimeout (30 s on OtOpcUa QA). The service now takes an optional IHostApplicationLifetime (DI supplies it in any generic host): ApplicationStopping is linked into the stream's token and ends the session cleanly, and a stream arriving once stopping has begun is refused Unavailable. Wire-compatible; no proto/schema change. Tests: open stream ends within 5 s of StopApplication, new stream refused Unavailable, control without a lifetime still running; both halves mutation-killed. 262/262. Source only, NOT published (still 0.3.7). Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
@@ -22,6 +23,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
private readonly ILogger<LocalDbSyncService> _logger;
|
||||
private readonly LocalDbMetrics? _metrics;
|
||||
private readonly SyncStatus? _status;
|
||||
private readonly CancellationToken _stopping;
|
||||
|
||||
// The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service).
|
||||
private readonly object _gate = new();
|
||||
@@ -48,10 +50,13 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
/// Creates the passive sync service. <paramref name="status"/> is taken as the public
|
||||
/// <see cref="ISyncStatus"/> (SyncStatus is internal and cannot appear in a public signature);
|
||||
/// DI always supplies the concrete SyncStatus the ctor recovers by cast.
|
||||
/// <paramref name="lifetime"/>, when present (any generic host supplies it), ends every open
|
||||
/// stream on <see cref="IHostApplicationLifetime.ApplicationStopping"/> — see <see cref="Sync"/>.
|
||||
/// </summary>
|
||||
public LocalDbSyncService(
|
||||
ILocalDb db, IOptions<ReplicationOptions> options, ILoggerFactory loggerFactory,
|
||||
LocalDbMetrics? metrics = null, ISyncStatus? status = null)
|
||||
LocalDbMetrics? metrics = null, ISyncStatus? status = null,
|
||||
IHostApplicationLifetime? lifetime = null)
|
||||
{
|
||||
_db = SyncSessionFactory.RequireSqlite(db, nameof(LocalDbSyncService));
|
||||
_options = options.Value;
|
||||
@@ -59,6 +64,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
_logger = loggerFactory.CreateLogger<LocalDbSyncService>();
|
||||
_metrics = metrics;
|
||||
_status = status as SyncStatus;
|
||||
_stopping = lifetime?.ApplicationStopping ?? CancellationToken.None;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(_options.ApiKey))
|
||||
{
|
||||
@@ -90,6 +96,14 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
/// that cannot be evicted within <see cref="TakeoverTimeout"/> is the one case where the old
|
||||
/// answer is still right — the newcomer refuses itself rather than double-running.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Host shutdown ends the stream.</b> The call token fires only when the peer goes away or the
|
||||
/// server aborts the request — and Kestrel aborts only at the END of its graceful drain, so through
|
||||
/// 0.3.7 every graceful stop of the passive node waited out the host's whole <c>ShutdownTimeout</c>
|
||||
/// (30 s by default) for a stream that never ends on its own (lmxopcua#906). The stream is now also
|
||||
/// cancelled by <see cref="IHostApplicationLifetime.ApplicationStopping"/>, and a stream that
|
||||
/// arrives once stopping has begun is refused <c>Unavailable</c> (the peer retries with backoff).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public override async Task Sync(
|
||||
IAsyncStreamReader<SyncMessage> requestStream,
|
||||
@@ -108,7 +122,10 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
// path — including a session fault for a local (non-transport) reason, where the call token
|
||||
// never fires and the pump would otherwise keep reading a stream whose handler has exited.
|
||||
// It is also the handle a later call cancels to displace this one.
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken);
|
||||
if (_stopping.IsCancellationRequested)
|
||||
throw new RpcException(new Status(StatusCode.Unavailable, "this node is shutting down"));
|
||||
|
||||
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, _stopping);
|
||||
var mine = new Registration(linkedCts);
|
||||
|
||||
try
|
||||
@@ -142,6 +159,11 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase
|
||||
{
|
||||
// Client disconnect / server shutdown: the normal end of a long-lived streaming RPC.
|
||||
}
|
||||
catch (OperationCanceledException) when (_stopping.IsCancellationRequested)
|
||||
{
|
||||
// Host stopping: end the stream now rather than hold the server's graceful drain open.
|
||||
_logger.LogInformation("Sync stream ended: host is stopping.");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Displaced by a newer stream: an orderly handover, not a failure of this node.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Threading.Channels;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
@@ -35,7 +36,7 @@ public sealed class SyncStreamGuardTests : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private LocalDbSyncService NewService()
|
||||
private LocalDbSyncService NewService(IHostApplicationLifetime? lifetime = null)
|
||||
{
|
||||
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
|
||||
_paths.Add(path);
|
||||
@@ -47,7 +48,8 @@ public sealed class SyncStreamGuardTests : IDisposable
|
||||
return new LocalDbSyncService(
|
||||
db,
|
||||
Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }),
|
||||
NullLoggerFactory.Instance);
|
||||
NullLoggerFactory.Instance,
|
||||
lifetime: lifetime);
|
||||
}
|
||||
|
||||
private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct);
|
||||
@@ -115,6 +117,67 @@ public sealed class SyncStreamGuardTests : IDisposable
|
||||
await SwallowAsync(third);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostStopping_EndsTheOpenStream_Cleanly()
|
||||
{
|
||||
// lmxopcua#906: the call token fires only at the END of Kestrel's graceful drain, so a stream
|
||||
// that observed nothing else held every graceful stop of the passive node for the whole
|
||||
// host ShutdownTimeout (30 s on QA). ApplicationStopping must end it now — and as an orderly
|
||||
// end of the call, not a fault.
|
||||
var lifetime = new FakeLifetime();
|
||||
var service = NewService(lifetime);
|
||||
|
||||
using var callCts = new CancellationTokenSource(); // never cancelled: the peer is still there
|
||||
var output = new RecordingWriter();
|
||||
var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token));
|
||||
await output.NextAsync(Timeout);
|
||||
Assert.False(call.IsCompleted);
|
||||
|
||||
lifetime.StopApplication();
|
||||
|
||||
await call.WaitAsync(TimeSpan.FromSeconds(5)); // completes, and without throwing
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HostStopping_RefusesANewStream_Unavailable()
|
||||
{
|
||||
// The peer redials the moment its stream ends; a node already stopping must not admit it.
|
||||
var lifetime = new FakeLifetime();
|
||||
var service = NewService(lifetime);
|
||||
lifetime.StopApplication();
|
||||
|
||||
var ex = await Assert.ThrowsAsync<RpcException>(() =>
|
||||
service.Sync(new SilentReader(), new RecordingWriter(), ContextFor(CancellationToken.None)));
|
||||
Assert.Equal(StatusCode.Unavailable, ex.StatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WithoutALifetime_TheStreamRunsUntilItsCallEnds()
|
||||
{
|
||||
// Control for the two tests above: the same open stream, with no lifetime, is still running
|
||||
// after the same wait — so their prompt end is the lifetime's doing, not the harness's.
|
||||
var service = NewService();
|
||||
using var callCts = new CancellationTokenSource();
|
||||
var output = new RecordingWriter();
|
||||
var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token));
|
||||
await output.NextAsync(Timeout);
|
||||
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
||||
Assert.False(call.IsCompleted);
|
||||
|
||||
callCts.Cancel();
|
||||
await call.WaitAsync(Timeout);
|
||||
}
|
||||
|
||||
private sealed class FakeLifetime : IHostApplicationLifetime
|
||||
{
|
||||
private readonly CancellationTokenSource _stopping = new();
|
||||
public CancellationToken ApplicationStarted => CancellationToken.None;
|
||||
public CancellationToken ApplicationStopping => _stopping.Token;
|
||||
public CancellationToken ApplicationStopped => CancellationToken.None;
|
||||
public void StopApplication() => _stopping.Cancel();
|
||||
}
|
||||
|
||||
private static async Task SwallowAsync(Task task)
|
||||
{
|
||||
try { await task.WaitAsync(Timeout); }
|
||||
|
||||
Reference in New Issue
Block a user