feat(localdb): telemetry meters + ISyncStatus

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 00:03:39 -04:00
parent 8befebd476
commit be4ca76f5f
9 changed files with 765 additions and 8 deletions
@@ -79,6 +79,21 @@ internal sealed class OplogStore(ILocalDb db, ReplicationOptions options, Func<D
        return rows[0];
    }

+    /// <summary>
+    /// The unacked backlog read synchronously on a dedicated short-lived connection, for the
+    /// metric-observation callbacks (gauge scrape / status poll). Cheap enough at that frequency;
+    /// mirrors <see cref="GetOplogDepthAsync"/>'s query. May throw if the db is disposing — callers
+    /// on the observation path swallow and drop the sample.
+    /// </summary>
+    public long GetOplogDepthSync()
+    {
+        using var conn = db.CreateConnection();
+        using var cmd = conn.CreateCommand();
+        cmd.CommandText =
+            "SELECT COUNT(*) FROM __localdb_oplog WHERE seq > (SELECT last_acked_seq FROM __localdb_peer_state WHERE id = 1)";
+        return (long)cmd.ExecuteScalar()!;
+    }
+
    /// <summary>The highest oplog seq present, or 0 when the oplog is empty.</summary>
    public async Task<long> GetMaxSeqAsync(CancellationToken ct = default)
    {
@@ -31,6 +31,8 @@ internal sealed class SyncSession
    private readonly ReplicationOptions _options;
    private readonly ILogger _logger;
    private readonly Func<DateTimeOffset> _utcNow;
+    private readonly LocalDbMetrics? _metrics;
+    private readonly SyncStatus? _status;

    // Highest oplog seq handed to the wire this session; the pump reads strictly above it so a batch
    // is never re-sent before its ack (the reliable ordered stream delivers exactly once in-session).
@@ -45,7 +47,7 @@ internal sealed class SyncSession

    public SyncSession(
        SqliteLocalDb db, OplogStore store, LwwApplier applier, ReplicationOptions options, ILogger logger,
-        Func<DateTimeOffset>? utcNow = null)
+        Func<DateTimeOffset>? utcNow = null, LocalDbMetrics? metrics = null, SyncStatus? status = null)
    {
        _db = db;
        _store = store;
@@ -53,6 +55,8 @@ internal sealed class SyncSession
        _options = options;
        _logger = logger;
        _utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
+        _metrics = metrics;
+        _status = status;
    }

    /// <summary>
@@ -140,6 +144,7 @@ internal sealed class SyncSession

        await _store.SetPeerNodeIdAsync(peerHandshake.NodeId, ct);
        PeerNodeId = peerHandshake.NodeId;
+        _status?.SetPeerNodeId(peerHandshake.NodeId);
        _sentThruSeq = peerHandshake.LastAppliedRemoteSeq;
        Interlocked.Exchange(ref _peerAckedSeq, peerHandshake.LastAppliedRemoteSeq);

@@ -164,6 +169,7 @@ internal sealed class SyncSession
            if (SnapshotSender is null)
                throw new NotSupportedException("snapshot required but no snapshot sender configured");
            var snapshotAsOfSeq = await SnapshotSender(dataSend, ct);
+            _metrics?.RecordSnapshot("sent");
            // The snapshot already carries every row state up to as-of; skipping the pump past it
            // avoids re-sending tail deltas the peer would only discard via LWW.
            if (snapshotAsOfSeq > _sentThruSeq)
@@ -269,6 +275,7 @@ internal sealed class SyncSession
                                "Replication protocol error: SnapshotComplete with no active snapshot (missing SnapshotBegin).");
                        await SnapshotApplier!.OnCompleteAsync(msg.SnapshotComplete, ct);
                        _snapshotReceiving = false;
+                        _metrics?.RecordSnapshot("received");
                        break;
                    default:
                        throw new InvalidOperationException(
@@ -288,6 +295,7 @@ internal sealed class SyncSession
        var driftLimitMs = nowMs + (long)_options.MaxHlcDriftAhead.TotalMilliseconds;
        var toApply = new List<OplogEntryRecord>(batch.Entries.Count);
        var maxSeq = 0L;
+        var driftDeadLettered = 0;

        foreach (var proto in batch.Entries)
        {
@@ -310,6 +318,7 @@ internal sealed class SyncSession
                if (_options.FailClosedOnDrift)
                {
                    await DeadLetterDriftedAsync(entry, physicalMs, nowMs, now, ct);
+                    driftDeadLettered++;
                    continue;
                }
            }
@@ -317,6 +326,10 @@ internal sealed class SyncSession
        }

        var result = await _applier.ApplyBatchAsync(toApply, ct);
+        _metrics?.RecordApplied(result.Applied);
+        _metrics?.RecordConflictsDiscarded(result.Discarded);
+        _metrics?.RecordDeadLettered(result.DeadLettered + driftDeadLettered);
+        _status?.SetLastSyncUtc(now);
        var ackThru = Math.Max(result.AppliedThruSeq, maxSeq);
        if (ackThru > result.AppliedThruSeq)
            // Dead-lettered entries carried the batch's top seqs; advance the durable remote
@@ -20,12 +20,14 @@ internal static class SyncSessionFactory
            $"{consumer} requires the ILocalDb registered by AddZbLocalDb (a {nameof(SqliteLocalDb)}); " +
            $"got {db.GetType().FullName}.");

-    public static SyncSession Create(SqliteLocalDb db, ReplicationOptions options, ILoggerFactory loggerFactory)
+    public static SyncSession Create(
+        SqliteLocalDb db, ReplicationOptions options, ILoggerFactory loggerFactory,
+        LocalDbMetrics? metrics = null, SyncStatus? status = null)
    {
        var store = new OplogStore(db, options);
        var applier = new LwwApplier(db);
        var logger = loggerFactory.CreateLogger<SyncSession>();
-        var session = new SyncSession(db, store, applier, options, logger);
+        var session = new SyncSession(db, store, applier, options, logger, utcNow: null, metrics, status);

        var streamer = new SnapshotStreamer(db, store, options, loggerFactory.CreateLogger<SnapshotStreamer>());
        session.SnapshotSender = streamer.SendAsync;