dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(localdb): fault live sync session on cap-prune; keepalive + newest-wins sync guard (0.3.2, ScadaBridge arch-review P4-1 + P4-3)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-19 14:49:31 -04:00
parent dbcbc5bd41
commit f62b017cc7
13 changed files with 574 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/GrpcAdapterTests.cs
+15 -3
View File
@@ -70,8 +70,12 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
}
[Fact]
public async Task SecondConcurrentStream_Rejected()
public async Task SecondConcurrentStream_DisplacesTheIncumbent()
{
// Newest wins (0.3.2). The guard serializes sessions; it was never a fence — the auth
// interceptor is, and an authenticated same-key dialler IS the peer. Refusing the newcomer
// bought nothing and cost the pair its only route back from a half-open link, where the
// incumbent handler can hold the guard indefinitely on a dead stream.
var serverHost = await BuildServerAsync(NewDbPath(), apiKey: null, sink: null);
var serverDb = serverHost.Services.GetRequiredService<ILocalDb>();
var handler = serverHost.GetTestServer().CreateHandler();
@@ -85,11 +89,15 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
// Convergence proves the first stream's server handler is active and holding the single-stream guard.
await WaitForAsync(async () => (await ReadOrders(serverDb)).Count == 1, RunTimeout);
var attemptsBefore = bg.ConnectionAttempts;
var client = new LocalDbSync.LocalDbSyncClient(ChannelOver(handler));
using var call2 = client.Sync(cancellationToken: cts.Token);
var ex = await Assert.ThrowsAsync<RpcException>(() => call2.ResponseStream.MoveNext(cts.Token));
Assert.Equal(StatusCode.AlreadyExists, ex.StatusCode);
Assert.True(await call2.ResponseStream.MoveNext(cts.Token));
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, call2.ResponseStream.Current.MsgCase);
// The incumbent really ended: the displaced client notices and dials again.
await WaitForAsync(() => Task.FromResult(bg.ConnectionAttempts > attemptsBefore), RunTimeout);
await bg.StopAsync(CancellationToken.None);
}
@@ -262,9 +270,13 @@ public sealed class GrpcAdapterTests : IAsyncLifetime
TombstoneRetention = TimeSpan.Zero,
ReconnectBackoffMax = TimeSpan.Zero,
MaxHlcDriftAhead = TimeSpan.Zero,
KeepAlivePingDelay = TimeSpan.Zero,
KeepAlivePingTimeout = TimeSpan.Zero,
});
Assert.True(result.Failed);
Assert.Contains(result.Failures, f => f.Contains("KeepAlivePingDelay", StringComparison.Ordinal));
Assert.Contains(result.Failures, f => f.Contains("KeepAlivePingTimeout", StringComparison.Ordinal));
Assert.Contains(result.Failures, f => f.Contains("FlushInterval", StringComparison.Ordinal));
Assert.Contains(result.Failures, f => f.Contains("MaxBatchSize", StringComparison.Ordinal));
Assert.Contains(result.Failures, f => f.Contains("MaxOplogRows", StringComparison.Ordinal));
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SnapshotResyncTests.cs
+93 -2
View File
@@ -125,11 +125,15 @@ public sealed class SnapshotResyncTests : IDisposable
// Completes the two-phase handshake against a real session: the peer never requests a snapshot.
private static async Task DriveHandshakeAsync(
SqliteLocalDb sessionDb, ChannelWriter<SyncMessage> toSession, ChannelReader<SyncMessage> fromSession,
CancellationToken ct, bool supportsPartialSnapshot = false)
CancellationToken ct, bool supportsPartialSnapshot = false, long lastAppliedRemoteSeq = 0)
{
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.Handshake, ct);
await toSession.WriteAsync(
new SyncMessage { Handshake = MatchingHandshake(sessionDb, "peer", supportsPartialSnapshot: supportsPartialSnapshot) }, ct);
new SyncMessage
{
Handshake = MatchingHandshake(
sessionDb, "peer", lastAppliedRemoteSeq, supportsPartialSnapshot),
}, ct);
await NextOfCaseAsync(fromSession, SyncMessage.MsgOneofCase.HandshakeAck, ct);
await toSession.WriteAsync(new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = "peer", SnapshotRequired = false } }, ct);
}
@@ -373,6 +377,93 @@ public sealed class SnapshotResyncTests : IDisposable
await SwallowAsync(runB);
}
[Fact]
public async Task CapPrune_IntoALiveSessionsUnsentRange_FaultsIt_AndTheReconnectSnapshots()
{
// P4-1. The pump reads `seq > sentThruSeq`, which resumes SILENTLY above a hole a cap prune
// punched — so it keeps shipping later deltas, the peer acks them, and the pair is durably
// divergent. `needs_snapshot` (set in the prune's own transaction) heals only a RECONNECT,
// and nothing forced one: on a stable LAN with a slow-but-alive peer the session, and the
// divergence, could last for days. Faulting is what turns the persisted flag into a snapshot.
var a = await NewSideAsync();
for (var i = 1; i <= 6; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
var (duplex, toSession, fromSession) = ScriptedPeer();
using var cts = new CancellationTokenSource(RunTimeout);
var run = a.Session.RunAsync(duplex, cts.Token);
// The peer holds everything so far: no snapshot is owed, and the pump idles at seq 6.
await DriveHandshakeAsync(a.Db, toSession, fromSession, cts.Token, lastAppliedRemoteSeq: 6);
// Write three more rows and prune all but the newest, in ONE transaction — which is what a
// cap prune is, and what makes this deterministic: the pump's reader can only ever see the
// before state (nothing above seq 6) or the after state (seqs 7 and 8 already gone). It can
// never catch rows 7/8 in between and legitimately advance past them.
await using (var tx = await a.Db.BeginTransactionAsync(cts.Token))
{
await tx.ExecuteAsync(
"INSERT INTO orders (id, sku, qty) VALUES (7,'PRUNED',7),(8,'PRUNED',8),(9,'KEPT',9)", null, cts.Token);
await tx.ExecuteAsync("UPDATE __localdb_peer_state SET needs_snapshot = 1 WHERE id = 1", null, cts.Token);
await tx.ExecuteAsync(
"DELETE FROM __localdb_oplog WHERE seq < (SELECT MAX(seq) FROM __localdb_oplog)", null, cts.Token);
await tx.CommitAsync(cts.Token);
}
var fault = await Assert.ThrowsAsync<LocalDbOplogGapException>(() => run);
Assert.Contains("6", fault.Message, StringComparison.Ordinal);
// And it faulted BEFORE shipping anything from above the hole: a delta for seq 9 would have
// moved the peer's watermark past rows 7 and 8, which is the divergence itself.
while (fromSession.TryRead(out var stray))
Assert.NotEqual(SyncMessage.MsgOneofCase.DeltaBatch, stray.MsgCase);
// The reconnect is what heals: the prune's needs_snapshot makes the handshake owe a full
// snapshot, which carries the pruned rows' current state to a real peer.
var a2 = NewSessionOver(a.Db);
var b = await NewSideAsync();
var (da, db) = DuplexPair();
var run2A = a2.Session.RunAsync(da, cts.Token);
var run2B = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 9, RunTimeout);
Assert.Equal(await ReadOrders(a.Db), await ReadOrders(b.Db));
cts.Cancel();
await SwallowAsync(run2A);
await SwallowAsync(run2B);
}
[Fact]
public async Task AckPruningToEmpty_DoesNotFaultALiveSession()
{
// The other half of the horizon check: ordinary ack-pruning deletes seq <= last_acked_seq,
// which by construction sits at or below the sent watermark, so a converged pair that has
// pruned its whole oplog must keep streaming. A gap test written against `needs_snapshot`,
// or one that treated any risen horizon as a fault, would kill healthy sessions here.
var a = await NewSideAsync();
var b = await NewSideAsync();
for (var i = 1; i <= 4; i++)
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (@id, 'A', @id)", new { id = i });
var (da, db) = DuplexPair();
using var cts = new CancellationTokenSource(RunTimeout);
var runA = a.Session.RunAsync(da, cts.Token);
var runB = b.Session.RunAsync(db, cts.Token);
await WaitForAsync(async () => (await ReadOrders(b.Db)).Count == 4, RunTimeout);
await WaitForAsync(async () => await OplogCount(a.Db) == 0, RunTimeout);
// Still live, still incremental: a post-prune write reaches B as an ordinary delta.
await a.Db.ExecuteAsync("INSERT INTO orders (id, sku, qty) VALUES (5, 'POST', 5)");
await WaitForAsync(async () => (await ReadOrders(b.Db)).Any(r => r.Id == 5), RunTimeout);
Assert.False(runA.IsFaulted, runA.Exception?.ToString());
cts.Cancel();
await SwallowAsync(runA);
await SwallowAsync(runB);
}
[Fact]
public async Task Snapshot_MergesNotTruncates()
{
ZB.MOM.WW.LocalDb/tests/ZB.MOM.WW.LocalDb.Tests/SyncStreamGuardTests.cs
+181
View File
@@ -0,0 +1,181 @@
using System.Threading.Channels;
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.LocalDb.Contracts;
using ZB.MOM.WW.LocalDb.Replication;
namespace ZB.MOM.WW.LocalDb.Tests;
/// <summary>
/// The passive service's single-stream guard under a HALF-OPEN link: the dialling node is gone, but
/// nothing on this side knows it, so the handler sits inside <c>MoveNext</c> forever holding the
/// guard. Through 0.3.1 that made the pair unrecoverable without a restart — every reconnect from
/// the rebooted peer was refused <c>AlreadyExists</c>, indefinitely on an idle pair, while this node
/// went on reporting itself connected. The guard is there to serialize sessions, not to fence out a
/// caller the auth interceptor already vouched for, so the newest dialler wins.
/// </summary>
public sealed class SyncStreamGuardTests : IDisposable
{
private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10);
private readonly List<string> _paths = [];
private readonly List<IDisposable> _disposables = [];
public void Dispose()
{
foreach (var d in _disposables)
d.Dispose();
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
foreach (var p in _paths)
{
if (File.Exists(p)) File.Delete(p);
if (File.Exists(p + "-wal")) File.Delete(p + "-wal");
if (File.Exists(p + "-shm")) File.Delete(p + "-shm");
}
}
private LocalDbSyncService NewService()
{
var path = Path.Combine(Path.GetTempPath(), Guid.NewGuid() + ".db");
_paths.Add(path);
var db = new Internal.SqliteLocalDb(new LocalDbOptions { Path = path });
_disposables.Add(db);
db.ExecuteAsync("CREATE TABLE orders (id INTEGER PRIMARY KEY, sku TEXT)").GetAwaiter().GetResult();
db.RegisterReplicated("orders");
return new LocalDbSyncService(
db,
Options.Create(new ReplicationOptions { FlushInterval = TimeSpan.FromMilliseconds(20) }),
NullLoggerFactory.Instance);
}
private static ServerCallContext ContextFor(CancellationToken ct) => new FakeCallContext(ct);
[Fact]
public async Task WedgedIncumbent_IsDisplacedByANewStream()
{
var service = NewService();
// Eviction is normally generous; the test only needs it to not be the thing under test.
service.TakeoverTimeout = Timeout;
// The incumbent's peer crashed: its stream will never deliver another message, and neither
// end times out on silence — a converged pair is legitimately quiet. The handler parks in
// the handshake's first read, holding the guard.
using var incumbentCts = new CancellationTokenSource();
var incumbentOut = new RecordingWriter();
var incumbent = service.Sync(new SilentReader(), incumbentOut, ContextFor(incumbentCts.Token));
// It really is running and really did claim the guard (its handshake is on the wire).
await incumbentOut.NextAsync(Timeout);
Assert.False(incumbent.IsCompleted);
// The rebooted peer dials again. Pre-fix this threw AlreadyExists and the pair stayed wedged.
using var newcomerCts = new CancellationTokenSource();
var newcomerOut = new RecordingWriter();
var newcomer = service.Sync(new SilentReader(), newcomerOut, ContextFor(newcomerCts.Token));
// The incumbent is torn down — gracefully, not as a fault of this node.
await incumbent.WaitAsync(Timeout);
// ...and the new stream is genuinely served: its own handshake goes out.
var handshake = await newcomerOut.NextAsync(Timeout);
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, handshake.MsgCase);
Assert.False(newcomer.IsCompleted);
newcomerCts.Cancel();
await SwallowAsync(newcomer);
}
[Fact]
public async Task DisplacedIncumbent_ReleasesTheGuard_SoAThirdStreamIsAlsoServed()
{
// Takeover must leave the guard in a state that can be taken over AGAIN — a handover that
// leaked the slot would simply move the wedge one reconnect down the line.
var service = NewService();
service.TakeoverTimeout = Timeout;
using var firstCts = new CancellationTokenSource();
var firstOut = new RecordingWriter();
var first = service.Sync(new SilentReader(), firstOut, ContextFor(firstCts.Token));
await firstOut.NextAsync(Timeout);
using var secondCts = new CancellationTokenSource();
var secondOut = new RecordingWriter();
var second = service.Sync(new SilentReader(), secondOut, ContextFor(secondCts.Token));
await first.WaitAsync(Timeout);
await secondOut.NextAsync(Timeout);
using var thirdCts = new CancellationTokenSource();
var thirdOut = new RecordingWriter();
var third = service.Sync(new SilentReader(), thirdOut, ContextFor(thirdCts.Token));
await second.WaitAsync(Timeout);
Assert.Equal(SyncMessage.MsgOneofCase.Handshake, (await thirdOut.NextAsync(Timeout)).MsgCase);
thirdCts.Cancel();
await SwallowAsync(third);
}
private static async Task SwallowAsync(Task task)
{
try { await task.WaitAsync(Timeout); }
catch (OperationCanceledException) { }
catch (RpcException) { }
}
/// <summary>
/// The minimum <see cref="ServerCallContext"/> the handler actually reads: its cancellation
/// token. (Grpc.Core.Api ships no test double, and the ASP.NET Core one is not reachable outside
/// a live call.)
/// </summary>
private sealed class FakeCallContext(CancellationToken ct) : ServerCallContext
{
protected override string MethodCore => "/localdb_sync.v1.LocalDbSync/Sync";
protected override string HostCore => "test";
protected override string PeerCore => "test";
protected override DateTime DeadlineCore => DateTime.MaxValue;
protected override Metadata RequestHeadersCore => [];
protected override CancellationToken CancellationTokenCore => ct;
protected override Metadata ResponseTrailersCore { get; } = [];
protected override Status StatusCore { get; set; }
protected override WriteOptions? WriteOptionsCore { get; set; }
protected override AuthContext AuthContextCore => new(null, new Dictionary<string, List<AuthProperty>>());
protected override ContextPropagationToken CreatePropagationTokenCore(ContextPropagationOptions? options) =>
throw new NotSupportedException();
protected override Task WriteResponseHeadersAsyncCore(Metadata responseHeaders) => Task.CompletedTask;
}
/// <summary>A stream that never delivers anything and never ends — a half-open link.</summary>
private sealed class SilentReader : IAsyncStreamReader<SyncMessage>
{
public SyncMessage Current => throw new InvalidOperationException("no message");
public async Task<bool> MoveNext(CancellationToken cancellationToken)
{
await Task.Delay(System.Threading.Timeout.Infinite, cancellationToken);
return false;
}
}
private sealed class RecordingWriter : IServerStreamWriter<SyncMessage>
{
private readonly Channel<SyncMessage> _written = Channel.CreateUnbounded<SyncMessage>();
public WriteOptions? WriteOptions { get; set; }
public Task WriteAsync(SyncMessage message) => WriteAsync(message, CancellationToken.None);
public Task WriteAsync(SyncMessage message, CancellationToken cancellationToken)
{
_written.Writer.TryWrite(message);
return Task.CompletedTask;
}
public async Task<SyncMessage> NextAsync(TimeSpan timeout)
{
using var cts = new CancellationTokenSource(timeout);
return await _written.Reader.ReadAsync(cts.Token);
}
}
}