fix(localdb): FailClosedOnDrift applies to snapshot rows (scadaproj#39)

The drift guard lived only in the delta path; SnapshotApplier applied every
row unchecked, so a future-stamped snapshot row was applied, observed by the
clock and persisted as last_seen_hlc - moving the receiver's HLC ahead across
restarts despite FailClosedOnDrift (LocalDb-004). SnapshotApplier now takes
ReplicationOptions and runs the same per-row check (warn once per snapshot;
dead-letter under FailClosedOnDrift via the shared HlcDriftGuard, with the
row's HLC kept out of last_seen_hlc).

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 15:01:01 -05:00
parent 15a6c7c710
commit c207573595
9 changed files with 258 additions and 33 deletions
@@ -0,0 +1,33 @@
using System.Globalization;
using ZB.MOM.WW.LocalDb.Internal;
namespace ZB.MOM.WW.LocalDb.Replication.Internal;
/// <summary>
/// The fail-closed disposition for an inbound row whose HLC runs further ahead of local UTC than
/// <see cref="ReplicationOptions.MaxHlcDriftAhead"/> allows. Shared by the delta path
/// (<see cref="SyncSession"/>) and the snapshot path (<see cref="SnapshotApplier"/>) so both write
/// the same dead-letter row (LocalDb-004).
/// </summary>
internal static class HlcDriftGuard
{
/// <summary>The marker every drift dead-letter's <c>error</c> starts with.</summary>
public const string ErrorPrefix = "HLC drift ahead";
public static Task DeadLetterAsync(
SqliteLocalDb db, OplogEntryRecord entry, long physicalMs, long nowMs, DateTimeOffset now,
TimeSpan maxDriftAhead, CancellationToken ct) =>
db.ExecuteAsync(
"INSERT INTO __localdb_dead_letter (received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
new
{
utc = now.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat, CultureInfo.InvariantCulture),
t = entry.TableName,
pk = entry.PkJson,
row = entry.RowJson,
hlc = entry.Hlc,
node = entry.NodeId,
err = $"{ErrorPrefix}: entry physical time {physicalMs} ms (Unix UTC) exceeds local now {nowMs} ms + MaxHlcDriftAhead {maxDriftAhead} (fail-closed).",
}, ct);
}
@@ -1,5 +1,6 @@
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb.Contracts;
using ZB.MOM.WW.LocalDb.Hlc;
using ZB.MOM.WW.LocalDb.Internal;
using ZB.MOM.WW.LocalDb.Registration;
@@ -246,9 +247,13 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
/// </summary>
internal sealed class SnapshotApplier : ISnapshotApplier
{
private readonly SqliteLocalDb _db;
private readonly LwwApplier _applier;
private readonly OplogStore _store;
private readonly ILogger _logger;
private readonly ReplicationOptions _options;
private readonly Func<DateTimeOffset> _utcNow;
private bool _driftWarned;
private long _asOfSeq;
private long _rows;
private long _maxHlc;
@@ -260,14 +265,27 @@ internal sealed class SnapshotApplier : ISnapshotApplier
// sender's debt clears only on the SnapshotAck, so the whole snapshot is re-streamed.
private List<OplogEntryRecord> _pending = [];
// db is part of the collaborator's canonical shape (mirrors the streamer) even though the merge
// reaches the database through the applier and store; kept for construction symmetry in the factory.
public SnapshotApplier(SqliteLocalDb db, LwwApplier applier, OplogStore store, ILogger logger)
/// <param name="db">The receiving database (dead-letter target for over-drift rows).</param>
/// <param name="applier">The shared LWW applier.</param>
/// <param name="store">The oplog/peer-state store.</param>
/// <param name="logger">Logger.</param>
/// <param name="options">
/// Supplies the drift guard (<see cref="ReplicationOptions.MaxHlcDriftAhead"/> /
/// <see cref="ReplicationOptions.FailClosedOnDrift"/>), applied to snapshot rows exactly as the
/// delta path applies it (LocalDb-004). Defaults to <see cref="ReplicationOptions"/>' defaults:
/// warn, never dead-letter.
/// </param>
/// <param name="utcNow">Test seam for the drift clock.</param>
public SnapshotApplier(
SqliteLocalDb db, LwwApplier applier, OplogStore store, ILogger logger,
ReplicationOptions? options = null, Func<DateTimeOffset>? utcNow = null)
{
_ = db;
_db = db;
_applier = applier;
_store = store;
_logger = logger;
_options = options ?? new ReplicationOptions();
_utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
}
public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct)
@@ -276,6 +294,7 @@ internal sealed class SnapshotApplier : ISnapshotApplier
_rows = 0;
_maxHlc = 0;
_pending = [];
_driftWarned = false;
// A non-empty table list scopes the snapshot; empty (which is all a pre-0.3.0 sender can
// emit) means every registered table. Read here rather than on complete because
// SnapshotComplete does not repeat it — the receive loop guarantees a Begin precedes both.
@@ -291,12 +310,43 @@ internal sealed class SnapshotApplier : ISnapshotApplier
if (batch.Rows.Count == 0)
return;
// The same drift guard the delta path runs (SyncSession.ApplyDeltaEntriesAsync). Without it a
// row stamped by a peer whose clock ran far ahead was applied, LwwApplier observed its HLC,
// and OnCompleteAsync persisted it as last_seen_hlc — every later local write on this node
// then carried the future stamp, across restarts, defeating FailClosedOnDrift (LocalDb-004).
var now = _utcNow();
var nowMs = now.ToUnixTimeMilliseconds();
var driftLimitMs = nowMs + (long)_options.MaxHlcDriftAhead.TotalMilliseconds;
var entries = new List<OplogEntryRecord>(batch.Rows.Count);
foreach (var row in batch.Rows)
{
var entry = new OplogEntryRecord(
0, batch.TableName, row.PkJson, row.HasRowJson ? row.RowJson : null, row.Hlc, row.NodeId, row.IsTombstone);
var physicalMs = HybridLogicalClock.PhysicalMs(row.Hlc);
if (physicalMs > driftLimitMs)
{
if (!_driftWarned)
{
_driftWarned = true;
_logger.LogWarning(
"Snapshot row HLC physical time {EntryMs} ms is ahead of local UTC {NowMs} ms by more than MaxHlcDriftAhead ({Drift}). Further drift in this snapshot is not re-logged.",
physicalMs, nowMs, _options.MaxHlcDriftAhead);
}
if (_options.FailClosedOnDrift)
{
// Skipped rows contribute nothing to _maxHlc, so last_seen_hlc stays honest.
await HlcDriftGuard.DeadLetterAsync(_db, entry, physicalMs, nowMs, now, _options.MaxHlcDriftAhead, ct);
continue;
}
}
if (row.Hlc > _maxHlc) _maxHlc = row.Hlc;
entries.Add(new OplogEntryRecord(
0, batch.TableName, row.PkJson, row.HasRowJson ? row.RowJson : null, row.Hlc, row.NodeId, row.IsTombstone));
entries.Add(entry);
}
if (entries.Count == 0)
{
_rows += batch.Rows.Count;
return;
}
var result = await _applier.ApplyBatchAsync(entries, ct, deferForeignKeyViolations: true);
_rows += batch.Rows.Count;
@@ -1,4 +1,3 @@
using System.Globalization;
using System.Threading.Channels;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb.Contracts;
@@ -649,19 +648,7 @@ internal sealed class SyncSession
// dead-letters the same entry again (at-least-once — a benign duplicate dead-letter row).
private Task DeadLetterDriftedAsync(
OplogEntryRecord entry, long physicalMs, long nowMs, DateTimeOffset now, CancellationToken ct) =>
_db.ExecuteAsync(
"INSERT INTO __localdb_dead_letter (received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
new
{
utc = now.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat, CultureInfo.InvariantCulture),
t = entry.TableName,
pk = entry.PkJson,
row = entry.RowJson,
hlc = entry.Hlc,
node = entry.NodeId,
err = $"HLC drift ahead: entry physical time {physicalMs} ms (Unix UTC) exceeds local now {nowMs} ms + MaxHlcDriftAhead {_options.MaxHlcDriftAhead} (fail-closed).",
}, ct);
HlcDriftGuard.DeadLetterAsync(_db, entry, physicalMs, nowMs, now, _options.MaxHlcDriftAhead, ct);
private async Task DrainSecondaryAsync(Task task)
{
@@ -31,7 +31,7 @@ internal static class SyncSessionFactory
var streamer = new SnapshotStreamer(db, store, options, loggerFactory.CreateLogger<SnapshotStreamer>());
session.SnapshotSender = streamer.SendAsync;
session.SnapshotApplier = new SnapshotApplier(db, applier, store, loggerFactory.CreateLogger<SnapshotApplier>());
session.SnapshotApplier = new SnapshotApplier(db, applier, store, loggerFactory.CreateLogger<SnapshotApplier>(), options);
return session;
}
}