dohertj2/scadaproj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(localdb): sync session concurrency fixes — two-lane writer, teardown fault fidelity, per-entry drift, gated snapshot seam

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-17 22:44:42 -04:00
parent 925623b878
commit 92ccbac8db
3 changed files with 367 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/Internal/SyncDuplex.cs
+3 -3
View File
@@ -6,9 +6,9 @@ namespace ZB.MOM.WW.LocalDb.Replication.Internal;
/// <summary>
/// The transport seam a <see cref="SyncSession"/> runs over: an outbound send delegate plus an
/// inbound message reader. Task 11 adapts a gRPC duplex stream to this; tests wire two in-memory
/// <see cref="Channel{T}"/> pairs crosswise. <see cref="Send"/> may be called concurrently by the
/// session's pump and receive loops — the session serializes those calls, so an adapter's writer
/// only ever sees one send at a time.
/// <see cref="Channel{T}"/> pairs crosswise. The session's single writer task is the only caller
/// of <see cref="Send"/>, so an adapter's stream writer never sees concurrent sends. Send MUST
/// honor its CancellationToken even when blocked on transport flow control (teardown relies on it).
/// </summary>
internal sealed class SyncDuplex
{
ZB.MOM.WW.LocalDb/src/ZB.MOM.WW.LocalDb.Replication/Internal/SyncSession.cs
+178 -66
View File
@@ -1,3 +1,5 @@
using System.Globalization;
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb.Contracts;
using ZB.MOM.WW.LocalDb.Hlc;
@@ -9,13 +11,20 @@ namespace ZB.MOM.WW.LocalDb.Replication.Internal;
/// The symmetric 2-node sync protocol as one state machine: both the gRPC service and the gRPC
/// client run this identical logic over a <see cref="SyncDuplex"/>. It performs the fail-closed
/// handshake, then runs a bidirectional steady state — an outbound change pump and an inbound
/// apply/ack loop — until the token is cancelled or the stream ends. Local writes never block on
/// the peer; every failure is a typed exception that tears down both loops and propagates.
/// apply/ack loop — until the token is cancelled or the stream ends. All outbound traffic flows
/// through a single writer task fed by two lanes: an unbounded CONTROL lane (handshake messages,
/// acks) and a bounded DATA lane (delta/snapshot batches). Receiving therefore never blocks on the
/// ability to write — acks always fit — while the bounded data lane is the pump's backpressure.
/// Local writes never block on the peer; the first fault tears down all loops and propagates.
/// </summary>
internal sealed class SyncSession
{
private const uint LibSchemaVersion = 1;
// Data-lane depth bounds outbound batch buffering; control-lane volume is bounded by the
// inbound batch rate (the peer's flow control), so its lane can safely be unbounded.
private const int DataLaneCapacity = 16;
private readonly SqliteLocalDb _db;
private readonly OplogStore _store;
private readonly LwwApplier _applier;
@@ -23,14 +32,12 @@ internal sealed class SyncSession
private readonly ILogger _logger;
private readonly Func<DateTimeOffset> _utcNow;
// gRPC stream writers are not safe for concurrent writes; the pump and receive loops both send,
// so every outbound message goes through this gate to serialize them into one writer.
private readonly SemaphoreSlim _sendGate = new(1, 1);
// Highest oplog seq handed to the wire this session; the pump reads strictly above it so a batch
// is never re-sent before its ack (the reliable ordered stream delivers exactly once in-session).
private long _sentThruSeq;
private long _peerAckedSeq;
private string? _peerNodeId;
private bool _driftWarned;
public SyncSession(
SqliteLocalDb db, OplogStore store, LwwApplier applier, ReplicationOptions options, ILogger logger,
@@ -44,24 +51,74 @@ internal sealed class SyncSession
_utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
}
/// <summary>Invoked when the peer's HandshakeAck requests a snapshot before deltas. Task 12 fills this in.</summary>
public Func<SyncDuplex, CancellationToken, Task>? SnapshotSender { get; set; }
/// <summary>
/// Invoked when the peer's HandshakeAck requests a snapshot before deltas. Receives a gated
/// send delegate that enqueues on the bounded data lane (never the raw transport). Task 12 fills this in.
/// </summary>
public Func<Func<SyncMessage, CancellationToken, Task>, CancellationToken, Task>? SnapshotSender { get; set; }
/// <summary>Invoked on an inbound SnapshotBegin. Task 12 fills this in.</summary>
public Func<SnapshotBegin, SyncDuplex, CancellationToken, Task>? SnapshotReceiver { get; set; }
/// <summary>
/// Invoked on an inbound SnapshotBegin, with the triggering message and a gated data-lane send
/// delegate. Task 12 fills this in.
/// </summary>
public Func<SnapshotBegin, Func<SyncMessage, CancellationToken, Task>, CancellationToken, Task>? SnapshotReceiver { get; set; }
/// <summary>Highest seq the peer has acknowledged applying this session.</summary>
public long PeerAckedSeq => Interlocked.Read(ref _peerAckedSeq);
/// <summary>The peer node's identity, once the handshake has completed.</summary>
public string? PeerNodeId { get; private set; }
/// <summary>The peer node's identity, once the handshake has completed. Volatile: written on the session task, read from test/metric threads.</summary>
public string? PeerNodeId
{
get => Volatile.Read(ref _peerNodeId);
private set => Volatile.Write(ref _peerNodeId, value);
}
public async Task RunAsync(SyncDuplex duplex, CancellationToken ct)
{
var peerState = await _store.GetPeerStateAsync(ct);
var control = Channel.CreateUnbounded<SyncMessage>(new UnboundedChannelOptions { SingleReader = true });
var data = Channel.CreateBounded<SyncMessage>(new BoundedChannelOptions(DataLaneCapacity) { SingleReader = true });
Task GatedDataSend(SyncMessage message, CancellationToken token) => data.Writer.WriteAsync(message, token).AsTask();
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
var writer = WriterLoopAsync(duplex, control.Reader, data.Reader, linked.Token);
try
{
await HandshakeAsync(duplex, control.Writer, GatedDataSend, linked.Token);
var pump = PumpLoopAsync(data.Writer, linked.Token);
var receive = ReceiveLoopAsync(duplex, control.Writer, GatedDataSend, linked.Token);
var loops = new[] { pump, receive, writer };
// The first COMPLETED task is the primary outcome. Cancel the siblings, observe every
// one of them under a swallow (so no secondary exception can replace the primary and
// nothing is left running/unobserved), then surface the primary by awaiting it.
var finished = await Task.WhenAny(loops);
linked.Cancel();
foreach (var loop in loops)
if (!ReferenceEquals(loop, finished))
await DrainSecondaryAsync(loop);
await finished;
}
catch
{
// Handshake-phase faults (and the primary rethrow above) land here: the writer may
// still be running, so stop and observe it before the primary exception surfaces.
linked.Cancel();
await DrainSecondaryAsync(writer);
throw;
}
}
private async Task HandshakeAsync(
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane,
Func<SyncMessage, CancellationToken, Task> dataSend, CancellationToken ct)
{
var peerState = await _store.GetPeerStateAsync(ct);
var localHandshake = BuildHandshake(peerState);
await SendAsync(duplex, new SyncMessage { Handshake = localHandshake }, ct);
// Both peers send before reading, so the transport must buffer at least two in-flight
// messages (Handshake + HandshakeAck); gRPC's initial flow-control window covers that.
await controlLane.WriteAsync(new SyncMessage { Handshake = localHandshake }, ct);
var firstInbound = await ReceiveOneAsync(duplex, ct);
if (firstInbound.MsgCase != SyncMessage.MsgOneofCase.Handshake)
@@ -77,7 +134,8 @@ internal sealed class SyncSession
Interlocked.Exchange(ref _peerAckedSeq, peerHandshake.LastAppliedRemoteSeq);
var snapshotRequired = await ComputeSnapshotRequiredAsync(peerHandshake, peerState, ct);
await SendAsync(duplex, new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = _db.NodeId, SnapshotRequired = snapshotRequired } }, ct);
await controlLane.WriteAsync(
new SyncMessage { HandshakeAck = new HandshakeAck { NodeId = _db.NodeId, SnapshotRequired = snapshotRequired } }, ct);
var peerAckMsg = await ReceiveOneAsync(duplex, ct);
if (peerAckMsg.MsgCase != SyncMessage.MsgOneofCase.HandshakeAck)
@@ -88,39 +146,49 @@ internal sealed class SyncSession
{
if (SnapshotSender is null)
throw new NotSupportedException("snapshot required but no snapshot sender configured");
await SnapshotSender(duplex, ct);
await SnapshotSender(dataSend, ct);
}
await RunSteadyStateAsync(duplex, ct);
}
private async Task RunSteadyStateAsync(SyncDuplex duplex, CancellationToken ct)
// The ONLY caller of duplex.Send. Control (handshake + acks) always preempts data so the peer's
// inbound processing is never starved of its acks behind a backlog of DeltaBatches.
private static async Task WriterLoopAsync(
SyncDuplex duplex, ChannelReader<SyncMessage> control, ChannelReader<SyncMessage> data, CancellationToken ct)
{
using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct);
var pump = PumpLoopAsync(duplex, linked.Token);
var receive = ReceiveLoopAsync(duplex, linked.Token);
// Whichever loop completes first (fault or stream-end) drives teardown: cancel the sibling,
// drain it (swallowing only the cancellation we caused), then surface the primary outcome.
var finished = await Task.WhenAny(pump, receive);
linked.Cancel();
var other = finished == pump ? receive : pump;
try { await other; }
catch (OperationCanceledException) { }
await finished;
}
private async Task PumpLoopAsync(SyncDuplex duplex, CancellationToken ct)
{
while (!ct.IsCancellationRequested)
Task<bool>? controlWait = null;
Task<bool>? dataWait = null;
while (true)
{
ct.ThrowIfCancellationRequested();
if (control.TryRead(out var message) || data.TryRead(out message))
{
await duplex.Send(message, ct);
continue;
}
// At most one outstanding waiter per lane (SingleReader); a stale already-completed
// waiter only costs one extra TryRead pass.
controlWait ??= control.WaitToReadAsync(ct).AsTask();
dataWait ??= data.WaitToReadAsync(ct).AsTask();
var completed = await Task.WhenAny(controlWait, dataWait);
await completed;
if (controlWait.IsCompleted) controlWait = null;
if (dataWait.IsCompleted) dataWait = null;
}
}
private async Task PumpLoopAsync(ChannelWriter<SyncMessage> dataLane, CancellationToken ct)
{
while (true)
{
ct.ThrowIfCancellationRequested();
var batch = await _store.ReadBatchAboveAsync(_sentThruSeq, _options.MaxBatchSize, ct);
if (batch.Count > 0)
{
var delta = new DeltaBatch();
foreach (var entry in batch)
delta.Entries.Add(ToProto(entry));
await SendAsync(duplex, new SyncMessage { DeltaBatch = delta }, ct);
// Blocking here on a full data lane IS the pump's backpressure.
await dataLane.WriteAsync(new SyncMessage { DeltaBatch = delta }, ct);
_sentThruSeq = batch[^1].Seq;
continue;
}
@@ -129,7 +197,9 @@ internal sealed class SyncSession
}
}
private async Task ReceiveLoopAsync(SyncDuplex duplex, CancellationToken ct)
private async Task ReceiveLoopAsync(
SyncDuplex duplex, ChannelWriter<SyncMessage> controlLane,
Func<SyncMessage, CancellationToken, Task> dataSend, CancellationToken ct)
{
while (await duplex.Inbox.WaitToReadAsync(ct))
{
@@ -138,7 +208,7 @@ internal sealed class SyncSession
switch (msg.MsgCase)
{
case SyncMessage.MsgOneofCase.DeltaBatch:
await HandleDeltaBatchAsync(duplex, msg.DeltaBatch, ct);
await HandleDeltaBatchAsync(controlLane, msg.DeltaBatch, ct);
break;
case SyncMessage.MsgOneofCase.DeltaAck:
var acked = msg.DeltaAck.AppliedThruSeq;
@@ -148,7 +218,7 @@ internal sealed class SyncSession
case SyncMessage.MsgOneofCase.SnapshotBegin:
if (SnapshotReceiver is null)
throw new NotSupportedException("snapshot received but no snapshot receiver configured");
await SnapshotReceiver(msg.SnapshotBegin, duplex, ct);
await SnapshotReceiver(msg.SnapshotBegin, dataSend, ct);
break;
default:
throw new InvalidOperationException(
@@ -158,33 +228,82 @@ internal sealed class SyncSession
}
}
private async Task HandleDeltaBatchAsync(SyncDuplex duplex, DeltaBatch batch, CancellationToken ct)
private async Task HandleDeltaBatchAsync(ChannelWriter<SyncMessage> controlLane, DeltaBatch batch, CancellationToken ct)
{
var entries = new List<OplogEntryRecord>(batch.Entries.Count);
var maxHlc = 0L;
foreach (var e in batch.Entries)
{
entries.Add(FromProto(e));
if (e.Hlc > maxHlc) maxHlc = e.Hlc;
}
if (batch.Entries.Count == 0)
return;
if (entries.Count > 0)
var now = _utcNow();
var nowMs = now.ToUnixTimeMilliseconds();
var driftLimitMs = nowMs + (long)_options.MaxHlcDriftAhead.TotalMilliseconds;
var toApply = new List<OplogEntryRecord>(batch.Entries.Count);
var maxSeq = 0L;
foreach (var proto in batch.Entries)
{
var maxHlcMs = HybridLogicalClock.PhysicalMs(maxHlc);
var driftLimitMs = _utcNow().Add(_options.MaxHlcDriftAhead).ToUnixTimeMilliseconds();
if (maxHlcMs > driftLimitMs)
if (proto.Seq > maxSeq) maxSeq = proto.Seq;
var entry = FromProto(proto);
var physicalMs = HybridLogicalClock.PhysicalMs(entry.Hlc);
if (physicalMs > driftLimitMs)
{
_logger.LogWarning(
"Inbound delta HLC physical time {HlcMs} ms exceeds local clock + MaxHlcDriftAhead ({LimitMs} ms).",
maxHlcMs, driftLimitMs);
if (!_driftWarned)
{
// Rate-limited to one warning per session: drift is usually a sustained clock
// condition and would otherwise flood the log at the delta rate.
_driftWarned = true;
_logger.LogWarning(
"Inbound oplog entry HLC physical time {EntryMs} ms is ahead of local UTC {NowMs} ms by more than MaxHlcDriftAhead ({Drift}). Further drift this session is not re-logged.",
physicalMs, nowMs, _options.MaxHlcDriftAhead);
}
// Dead-letter is the fail-closed disposition: faulting the session instead would
// livelock on reconnect (the peer resends the same drifted entries forever).
if (_options.FailClosedOnDrift)
throw new InvalidOperationException(
$"Inbound HLC physical time {maxHlcMs} ms exceeds MaxHlcDriftAhead limit {driftLimitMs} ms; rejecting batch (fail-closed).");
{
await DeadLetterDriftedAsync(entry, physicalMs, nowMs, now, ct);
continue;
}
}
toApply.Add(entry);
}
var result = await _applier.ApplyBatchAsync(entries, ct);
await SendAsync(duplex, new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = result.AppliedThruSeq } }, ct);
var result = await _applier.ApplyBatchAsync(toApply, ct);
var ackThru = Math.Max(result.AppliedThruSeq, maxSeq);
if (ackThru > result.AppliedThruSeq)
// Dead-lettered entries carried the batch's top seqs; advance the durable remote
// watermark over them (MAX-guarded like the applier's) so a reconnect handshake does
// not make the peer resend what this node has already discarded.
await _db.ExecuteAsync(
"UPDATE __localdb_peer_state SET last_applied_remote_seq = MAX(last_applied_remote_seq, @seq) WHERE id = 1",
new { seq = ackThru }, ct);
await controlLane.WriteAsync(new SyncMessage { DeltaAck = new DeltaAck { AppliedThruSeq = ackThru } }, ct);
}
private Task DeadLetterDriftedAsync(
OplogEntryRecord entry, long physicalMs, long nowMs, DateTimeOffset now, CancellationToken ct) =>
_db.ExecuteAsync(
"INSERT INTO __localdb_dead_letter (received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
new
{
utc = now.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat, CultureInfo.InvariantCulture),
t = entry.TableName,
pk = entry.PkJson,
row = entry.RowJson,
hlc = entry.Hlc,
node = entry.NodeId,
err = $"HLC drift ahead: entry physical time {physicalMs} ms (Unix UTC) exceeds local now {nowMs} ms + MaxHlcDriftAhead {_options.MaxHlcDriftAhead} (fail-closed).",
}, ct);
private async Task DrainSecondaryAsync(Task task)
{
try
{
await task;
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Secondary sync-session task ended during teardown.");
}
}
private Handshake BuildHandshake(PeerState peerState)
@@ -237,13 +356,6 @@ internal sealed class SyncSession
return minSeq > 0 && peerHandshake.LastAppliedRemoteSeq < minSeq - 1;
}
private async Task SendAsync(SyncDuplex duplex, SyncMessage message, CancellationToken ct)
{
await _sendGate.WaitAsync(ct);
try { await duplex.Send(message, ct); }
finally { _sendGate.Release(); }
}
private static async Task<SyncMessage> ReceiveOneAsync(SyncDuplex duplex, CancellationToken ct)
{
if (await duplex.Inbox.WaitToReadAsync(ct) && duplex.Inbox.TryRead(out var message))