fix(localdb): FailClosedOnDrift applies to snapshot rows (scadaproj#39)
The drift guard lived only in the delta path; SnapshotApplier applied every row unchecked, so a future-stamped snapshot row was applied, observed by the clock and persisted as last_seen_hlc - moving the receiver's HLC ahead across restarts despite FailClosedOnDrift (LocalDb-004). SnapshotApplier now takes ReplicationOptions and runs the same per-row check (warn once per snapshot; dead-letter under FailClosedOnDrift via the shared HlcDriftGuard, with the row's HLC kept out of last_seen_hlc). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
using System.Globalization;
|
||||
using ZB.MOM.WW.LocalDb.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.LocalDb.Replication.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// The fail-closed disposition for an inbound row whose HLC runs further ahead of local UTC than
|
||||
/// <see cref="ReplicationOptions.MaxHlcDriftAhead"/> allows. Shared by the delta path
|
||||
/// (<see cref="SyncSession"/>) and the snapshot path (<see cref="SnapshotApplier"/>) so both write
|
||||
/// the same dead-letter row (LocalDb-004).
|
||||
/// </summary>
|
||||
internal static class HlcDriftGuard
|
||||
{
|
||||
/// <summary>The marker every drift dead-letter's <c>error</c> starts with.</summary>
|
||||
public const string ErrorPrefix = "HLC drift ahead";
|
||||
|
||||
public static Task DeadLetterAsync(
|
||||
SqliteLocalDb db, OplogEntryRecord entry, long physicalMs, long nowMs, DateTimeOffset now,
|
||||
TimeSpan maxDriftAhead, CancellationToken ct) =>
|
||||
db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_dead_letter (received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
|
||||
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
|
||||
new
|
||||
{
|
||||
utc = now.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat, CultureInfo.InvariantCulture),
|
||||
t = entry.TableName,
|
||||
pk = entry.PkJson,
|
||||
row = entry.RowJson,
|
||||
hlc = entry.Hlc,
|
||||
node = entry.NodeId,
|
||||
err = $"{ErrorPrefix}: entry physical time {physicalMs} ms (Unix UTC) exceeds local now {nowMs} ms + MaxHlcDriftAhead {maxDriftAhead} (fail-closed).",
|
||||
}, ct);
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
using ZB.MOM.WW.LocalDb.Hlc;
|
||||
using ZB.MOM.WW.LocalDb.Internal;
|
||||
using ZB.MOM.WW.LocalDb.Registration;
|
||||
|
||||
@@ -246,9 +247,13 @@ internal sealed class SnapshotStreamer(SqliteLocalDb db, OplogStore store, Repli
|
||||
/// </summary>
|
||||
internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
{
|
||||
private readonly SqliteLocalDb _db;
|
||||
private readonly LwwApplier _applier;
|
||||
private readonly OplogStore _store;
|
||||
private readonly ILogger _logger;
|
||||
private readonly ReplicationOptions _options;
|
||||
private readonly Func<DateTimeOffset> _utcNow;
|
||||
private bool _driftWarned;
|
||||
private long _asOfSeq;
|
||||
private long _rows;
|
||||
private long _maxHlc;
|
||||
@@ -260,14 +265,27 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
// sender's debt clears only on the SnapshotAck, so the whole snapshot is re-streamed.
|
||||
private List<OplogEntryRecord> _pending = [];
|
||||
|
||||
// db is part of the collaborator's canonical shape (mirrors the streamer) even though the merge
|
||||
// reaches the database through the applier and store; kept for construction symmetry in the factory.
|
||||
public SnapshotApplier(SqliteLocalDb db, LwwApplier applier, OplogStore store, ILogger logger)
|
||||
/// <param name="db">The receiving database (dead-letter target for over-drift rows).</param>
|
||||
/// <param name="applier">The shared LWW applier.</param>
|
||||
/// <param name="store">The oplog/peer-state store.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
/// <param name="options">
|
||||
/// Supplies the drift guard (<see cref="ReplicationOptions.MaxHlcDriftAhead"/> /
|
||||
/// <see cref="ReplicationOptions.FailClosedOnDrift"/>), applied to snapshot rows exactly as the
|
||||
/// delta path applies it (LocalDb-004). Defaults to <see cref="ReplicationOptions"/>' defaults:
|
||||
/// warn, never dead-letter.
|
||||
/// </param>
|
||||
/// <param name="utcNow">Test seam for the drift clock.</param>
|
||||
public SnapshotApplier(
|
||||
SqliteLocalDb db, LwwApplier applier, OplogStore store, ILogger logger,
|
||||
ReplicationOptions? options = null, Func<DateTimeOffset>? utcNow = null)
|
||||
{
|
||||
_ = db;
|
||||
_db = db;
|
||||
_applier = applier;
|
||||
_store = store;
|
||||
_logger = logger;
|
||||
_options = options ?? new ReplicationOptions();
|
||||
_utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
|
||||
}
|
||||
|
||||
public Task OnBeginAsync(SnapshotBegin begin, CancellationToken ct)
|
||||
@@ -276,6 +294,7 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
_rows = 0;
|
||||
_maxHlc = 0;
|
||||
_pending = [];
|
||||
_driftWarned = false;
|
||||
// A non-empty table list scopes the snapshot; empty (which is all a pre-0.3.0 sender can
|
||||
// emit) means every registered table. Read here rather than on complete because
|
||||
// SnapshotComplete does not repeat it — the receive loop guarantees a Begin precedes both.
|
||||
@@ -291,12 +310,43 @@ internal sealed class SnapshotApplier : ISnapshotApplier
|
||||
if (batch.Rows.Count == 0)
|
||||
return;
|
||||
|
||||
// The same drift guard the delta path runs (SyncSession.ApplyDeltaEntriesAsync). Without it a
|
||||
// row stamped by a peer whose clock ran far ahead was applied, LwwApplier observed its HLC,
|
||||
// and OnCompleteAsync persisted it as last_seen_hlc — every later local write on this node
|
||||
// then carried the future stamp, across restarts, defeating FailClosedOnDrift (LocalDb-004).
|
||||
var now = _utcNow();
|
||||
var nowMs = now.ToUnixTimeMilliseconds();
|
||||
var driftLimitMs = nowMs + (long)_options.MaxHlcDriftAhead.TotalMilliseconds;
|
||||
|
||||
var entries = new List<OplogEntryRecord>(batch.Rows.Count);
|
||||
foreach (var row in batch.Rows)
|
||||
{
|
||||
var entry = new OplogEntryRecord(
|
||||
0, batch.TableName, row.PkJson, row.HasRowJson ? row.RowJson : null, row.Hlc, row.NodeId, row.IsTombstone);
|
||||
var physicalMs = HybridLogicalClock.PhysicalMs(row.Hlc);
|
||||
if (physicalMs > driftLimitMs)
|
||||
{
|
||||
if (!_driftWarned)
|
||||
{
|
||||
_driftWarned = true;
|
||||
_logger.LogWarning(
|
||||
"Snapshot row HLC physical time {EntryMs} ms is ahead of local UTC {NowMs} ms by more than MaxHlcDriftAhead ({Drift}). Further drift in this snapshot is not re-logged.",
|
||||
physicalMs, nowMs, _options.MaxHlcDriftAhead);
|
||||
}
|
||||
if (_options.FailClosedOnDrift)
|
||||
{
|
||||
// Skipped rows contribute nothing to _maxHlc, so last_seen_hlc stays honest.
|
||||
await HlcDriftGuard.DeadLetterAsync(_db, entry, physicalMs, nowMs, now, _options.MaxHlcDriftAhead, ct);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (row.Hlc > _maxHlc) _maxHlc = row.Hlc;
|
||||
entries.Add(new OplogEntryRecord(
|
||||
0, batch.TableName, row.PkJson, row.HasRowJson ? row.RowJson : null, row.Hlc, row.NodeId, row.IsTombstone));
|
||||
entries.Add(entry);
|
||||
}
|
||||
if (entries.Count == 0)
|
||||
{
|
||||
_rows += batch.Rows.Count;
|
||||
return;
|
||||
}
|
||||
var result = await _applier.ApplyBatchAsync(entries, ct, deferForeignKeyViolations: true);
|
||||
_rows += batch.Rows.Count;
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Globalization;
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.LocalDb.Contracts;
|
||||
@@ -649,19 +648,7 @@ internal sealed class SyncSession
|
||||
// dead-letters the same entry again (at-least-once — a benign duplicate dead-letter row).
|
||||
private Task DeadLetterDriftedAsync(
|
||||
OplogEntryRecord entry, long physicalMs, long nowMs, DateTimeOffset now, CancellationToken ct) =>
|
||||
_db.ExecuteAsync(
|
||||
"INSERT INTO __localdb_dead_letter (received_utc, table_name, pk_json, row_json, hlc, node_id, error) " +
|
||||
"VALUES (@utc, @t, @pk, @row, @hlc, @node, @err)",
|
||||
new
|
||||
{
|
||||
utc = now.UtcDateTime.ToString(OplogStore.TombstoneUtcFormat, CultureInfo.InvariantCulture),
|
||||
t = entry.TableName,
|
||||
pk = entry.PkJson,
|
||||
row = entry.RowJson,
|
||||
hlc = entry.Hlc,
|
||||
node = entry.NodeId,
|
||||
err = $"HLC drift ahead: entry physical time {physicalMs} ms (Unix UTC) exceeds local now {nowMs} ms + MaxHlcDriftAhead {_options.MaxHlcDriftAhead} (fail-closed).",
|
||||
}, ct);
|
||||
HlcDriftGuard.DeadLetterAsync(_db, entry, physicalMs, nowMs, now, _options.MaxHlcDriftAhead, ct);
|
||||
|
||||
private async Task DrainSecondaryAsync(Task task)
|
||||
{
|
||||
|
||||
@@ -31,7 +31,7 @@ internal static class SyncSessionFactory
|
||||
|
||||
var streamer = new SnapshotStreamer(db, store, options, loggerFactory.CreateLogger<SnapshotStreamer>());
|
||||
session.SnapshotSender = streamer.SendAsync;
|
||||
session.SnapshotApplier = new SnapshotApplier(db, applier, store, loggerFactory.CreateLogger<SnapshotApplier>());
|
||||
session.SnapshotApplier = new SnapshotApplier(db, applier, store, loggerFactory.CreateLogger<SnapshotApplier>(), options);
|
||||
return session;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user