diff --git a/ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/LocalDbSyncService.cs b/ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/LocalDbSyncService.cs index 767b84c..b76913b 100644 --- a/ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/LocalDbSyncService.cs +++ b/ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/LocalDbSyncService.cs @@ -1,6 +1,7 @@ using System.Security.Cryptography; using System.Text; using Grpc.Core; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using ZB.MOM.WW.LocalDb.Contracts; @@ -22,6 +23,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase private readonly ILogger _logger; private readonly LocalDbMetrics? _metrics; private readonly SyncStatus? _status; + private readonly CancellationToken _stopping; // The stream currently holding the guard, or null when idle. Guarded by _gate (singleton service). private readonly object _gate = new(); @@ -48,10 +50,13 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase /// Creates the passive sync service. is taken as the public /// (SyncStatus is internal and cannot appear in a public signature); /// DI always supplies the concrete SyncStatus the ctor recovers by cast. + /// , when present (any generic host supplies it), ends every open + /// stream on — see . /// public LocalDbSyncService( ILocalDb db, IOptions options, ILoggerFactory loggerFactory, - LocalDbMetrics? metrics = null, ISyncStatus? status = null) + LocalDbMetrics? metrics = null, ISyncStatus? status = null, + IHostApplicationLifetime? lifetime = null) { _db = SyncSessionFactory.RequireSqlite(db, nameof(LocalDbSyncService)); _options = options.Value; @@ -59,6 +64,7 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase _logger = loggerFactory.CreateLogger(); _metrics = metrics; _status = status as SyncStatus; + _stopping = lifetime?.ApplicationStopping ?? CancellationToken.None; if (string.IsNullOrWhiteSpace(_options.ApiKey)) { @@ -90,6 +96,14 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase /// that cannot be evicted within is the one case where the old /// answer is still right — the newcomer refuses itself rather than double-running. /// + /// + /// Host shutdown ends the stream. The call token fires only when the peer goes away or the + /// server aborts the request — and Kestrel aborts only at the END of its graceful drain, so through + /// 0.3.7 every graceful stop of the passive node waited out the host's whole ShutdownTimeout + /// (30 s by default) for a stream that never ends on its own (lmxopcua#906). The stream is now also + /// cancelled by , and a stream that + /// arrives once stopping has begun is refused Unavailable (the peer retries with backoff). + /// /// public override async Task Sync( IAsyncStreamReader requestStream, @@ -108,7 +122,10 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase // path — including a session fault for a local (non-transport) reason, where the call token // never fires and the pump would otherwise keep reading a stream whose handler has exited. // It is also the handle a later call cancels to displace this one. - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken); + if (_stopping.IsCancellationRequested) + throw new RpcException(new Status(StatusCode.Unavailable, "this node is shutting down")); + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, _stopping); var mine = new Registration(linkedCts); try @@ -142,6 +159,11 @@ public sealed class LocalDbSyncService : LocalDbSync.LocalDbSyncBase { // Client disconnect / server shutdown: the normal end of a long-lived streaming RPC. } + catch (OperationCanceledException) when (_stopping.IsCancellationRequested) + { + // Host stopping: end the stream now rather than hold the server's graceful drain open. + _logger.LogInformation("Sync stream ended: host is stopping."); + } catch (OperationCanceledException) { // Displaced by a newer stream: an orderly handover, not a failure of this node. diff --git a/ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncStreamGuardTests.cs b/ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncStreamGuardTests.cs index 30a9d69..1db7e7f 100644 --- a/ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncStreamGuardTests.cs +++ b/ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncStreamGuardTests.cs @@ -1,5 +1,6 @@ using System.Threading.Channels; using Grpc.Core; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using ZB.MOM.WW.LocalDb.Contracts; @@ -35,7 +36,7 @@ public sealed class SyncStreamGuardTests : IDisposable } } - private LocalDbSyncService NewService() + private LocalDbSyncService NewService(IHostApplicationLifetime? lifetime = null) { var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db"); _paths.Add(path); @@ -47,7 +48,8 @@ public sealed class SyncStreamGuardTests : IDisposable return new LocalDbSyncService( db, Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }), - NullLoggerFactory.Instance); + NullLoggerFactory.Instance, + lifetime: lifetime); } private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct); @@ -115,6 +117,67 @@ public sealed class SyncStreamGuardTests : IDisposable await SwallowAsync(third); } + [Fact] + public async Task HostStopping_EndsTheOpenStream_Cleanly() + { + // lmxopcua#906: the call token fires only at the END of Kestrel's graceful drain, so a stream + // that observed nothing else held every graceful stop of the passive node for the whole + // host ShutdownTimeout (30 s on QA). ApplicationStopping must end it now — and as an orderly + // end of the call, not a fault. + var lifetime = new FakeLifetime(); + var service = NewService(lifetime); + + using var callCts = new CancellationTokenSource(); // never cancelled: the peer is still there + var output = new RecordingWriter(); + var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token)); + await output.NextAsync(Timeout); + Assert.False(call.IsCompleted); + + lifetime.StopApplication(); + + await call.WaitAsync(TimeSpan.FromSeconds(5)); // completes, and without throwing + } + + [Fact] + public async Task HostStopping_RefusesANewStream_Unavailable() + { + // The peer redials the moment its stream ends; a node already stopping must not admit it. + var lifetime = new FakeLifetime(); + var service = NewService(lifetime); + lifetime.StopApplication(); + + var ex = await Assert.ThrowsAsync(() => + service.Sync(new SilentReader(), new RecordingWriter(), ContextFor(CancellationToken.None))); + Assert.Equal(StatusCode.Unavailable, ex.StatusCode); + } + + [Fact] + public async Task WithoutALifetime_TheStreamRunsUntilItsCallEnds() + { + // Control for the two tests above: the same open stream, with no lifetime, is still running + // after the same wait — so their prompt end is the lifetime's doing, not the harness's. + var service = NewService(); + using var callCts = new CancellationTokenSource(); + var output = new RecordingWriter(); + var call = service.Sync(new SilentReader(), output, ContextFor(callCts.Token)); + await output.NextAsync(Timeout); + + await Task.Delay(TimeSpan.FromMilliseconds(500)); + Assert.False(call.IsCompleted); + + callCts.Cancel(); + await call.WaitAsync(Timeout); + } + + private sealed class FakeLifetime : IHostApplicationLifetime + { + private readonly CancellationTokenSource _stopping = new(); + public CancellationToken ApplicationStarted => CancellationToken.None; + public CancellationToken ApplicationStopping => _stopping.Token; + public CancellationToken ApplicationStopped => CancellationToken.None; + public void StopApplication() => _stopping.Cancel(); + } + private static async Task SwallowAsync(Task task) { try { await task.WaitAsync(Timeout); }