using System.Threading.Channels;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.LocalDb.Contracts;
using ZB.MOM.WW.LocalDb.Replication;
namespace ZB.MOM.WW.LocalDb.Tests;
///
/// The passive service's single-stream guard under a HALF-OPEN link: the dialling node is gone, but
/// nothing on this side knows it, so the handler sits inside MoveNext forever holding the
/// guard. Through 0.3.1 that made the pair unrecoverable without a restart — every reconnect from
/// the rebooted peer was refused AlreadyExists, indefinitely on an idle pair, while this node
/// went on reporting itself connected. The guard is there to serialize sessions, not to fence out a
/// caller the auth interceptor already vouched for, so the newest dialler wins.
///
public sealed class SyncStreamGuardTests : IDisposable
{
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10);
private readonly List _paths = [];
private readonly List _disposables = [];
public void Dispose()
{
foreach (var d in _disposables)
d.Dispose();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (var p in _paths)
{
if (File.Exists(p)) File.Delete(p);
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
}
}
private LocalDbSyncService NewService()
{
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
_paths.Add(path);
var db = new Internal.SqliteLocalDb(new LocalDbOptions { Path = path });
_disposables.Add(db);
db.ExecuteAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT)").GetAwaiter().GetResult();
db.RegisterReplicated("orders");
return new LocalDbSyncService(
db,
Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }),
NullLoggerFactory.Instance);
}
private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct);
[Fact]
public async Task WedgedIncumbent_IsDisplacedByANewStream()
{
var service = NewService();
// Eviction is normally generous; the test only needs it to not be the thing under test.
service.TakeoverTimeout = Timeout;
// The incumbent's peer crashed: its stream will never deliver another message, and neither
// end times out on silence — a converged pair is legitimately quiet. The handler parks in
// the handshake's first read, holding the guard.
using var incumbentCts = new CancellationTokenSource();
var incumbentOut = new RecordingWriter();
var incumbent = service.Sync(new SilentReader(), incumbentOut, ContextFor(incumbentCts.Token));
// It really is running and really did claim the guard (its handshake is on the wire).
await incumbentOut.NextAsync(Timeout);
Assert.False(incumbent.IsCompleted);
// The rebooted peer dials again. Pre-fix this threw AlreadyExists and the pair stayed wedged.
using var newcomerCts = new CancellationTokenSource();
var newcomerOut = new RecordingWriter();
var newcomer = service.Sync(new SilentReader(), newcomerOut, ContextFor(newcomerCts.Token));
// The incumbent is torn down — gracefully, not as a fault of this node.
await incumbent.WaitAsync(Timeout);
// ...and the new stream is genuinely served: its own handshake goes out.
var handshake = await newcomerOut.NextAsync(Timeout);
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, handshake.MsgCase);
Assert.False(newcomer.IsCompleted);
newcomerCts.Cancel();
await SwallowAsync(newcomer);
}
[Fact]
public async Task DisplacedIncumbent_ReleasesTheGuard_SoAThirdStreamIsAlsoServed()
{
// Takeover must leave the guard in a state that can be taken over AGAIN — a handover that
// leaked the slot would simply move the wedge one reconnect down the line.
var service = NewService();
service.TakeoverTimeout = Timeout;
using var firstCts = new CancellationTokenSource();
var firstOut = new RecordingWriter();
var first = service.Sync(new SilentReader(), firstOut, ContextFor(firstCts.Token));
await firstOut.NextAsync(Timeout);
using var secondCts = new CancellationTokenSource();
var secondOut = new RecordingWriter();
var second = service.Sync(new SilentReader(), secondOut, ContextFor(secondCts.Token));
await first.WaitAsync(Timeout);
await secondOut.NextAsync(Timeout);
using var thirdCts = new CancellationTokenSource();
var thirdOut = new RecordingWriter();
var third = service.Sync(new SilentReader(), thirdOut, ContextFor(thirdCts.Token));
await second.WaitAsync(Timeout);
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, (await thirdOut.NextAsync(Timeout)).MsgCase);
thirdCts.Cancel();
await SwallowAsync(third);
}
private static async Task SwallowAsync(Task task)
{
try { await task.WaitAsync(Timeout); }
catch (OperationCanceledException) { }
catch (RpcException) { }
}
///
/// The minimum the handler actually reads: its cancellation
/// token. (Grpc.Core.Api ships no test double, and the ASP.NET Core one is not reachable outside
/// a live call.)
///
private sealed class FakeCallContext(CancellationToken ct) : ServerCallContext
{
protected override string MethodCore => "/localdb_sync.v1.LocalDbSync/Sync";
protected override string HostCore => "test";
protected override string PeerCore => "test";
protected override DateTime DeadlineCore => DateTime.MaxValue;
protected override Metadata RequestHeadersCore => [];
protected override CancellationToken CancellationTokenCore => ct;
protected override Metadata ResponseTrailersCore { get; } = [];
protected override Status StatusCore { get; set; }
protected override WriteOptions? WriteOptionsCore { get; set; }
protected override AuthContext AuthContextCore => new(null, new Dictionary>());
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options) =>
throw new NotSupportedException();
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
}
/// A stream that never delivers anything and never ends — a half-open link.
private sealed class SilentReader : IAsyncStreamReader
{
public SyncMessage Current => throw new InvalidOperationException("no message");
public async Task MoveNext(CancellationToken cancellationToken)
{
await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken);
return false;
}
}
private sealed class RecordingWriter : IServerStreamWriter
{
private readonly Channel _written = Channel.CreateUnbounded();
public WriteOptions? WriteOptions { get; set; }
public Task WriteAsync(SyncMessage message) => WriteAsync(message, CancellationToken.None);
public Task WriteAsync(SyncMessage message, CancellationToken cancellationToken)
{
_written.Writer.TryWrite(message);
return Task.CompletedTask;
}
public async Task NextAsync(TimeSpan timeout)
{
using var cts = new CancellationTokenSource(timeout);
return await _written.Reader.ReadAsync(cts.Token);
}
}
}