feat(localdb): telemetry meters + ISyncStatus

Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
2026-07-18 00:03:39 -04:00
parent 8befebd476
commit be4ca76f5f
9 changed files with 765 additions and 8 deletions
@@ -0,0 +1,104 @@
+using System.Diagnostics.Metrics;
+
+namespace ZB.MOM.WW.LocalDb.Replication;
+
+/// <summary>
+/// The replication engine's OpenTelemetry instruments, under the single <c>zb.mom.ww.localdb</c>
+/// meter. Optional throughout: the sessions and adapters hold a nullable reference and every call
+/// is null-guarded, so replication runs identically whether or not this is registered. Instrument
+/// names follow the family metric conventions (lower-case, dot-separated, seconds for durations).
+/// </summary>
+public sealed class LocalDbMetrics : IDisposable
+{
+    /// <summary>The single meter name every replication instrument is published under.</summary>
+    public const string MeterName = "zb.mom.ww.localdb";
+
+    private readonly Meter _meter;
+    private readonly Counter<long> _applied;
+    private readonly Counter<long> _conflictsDiscarded;
+    private readonly Counter<long> _deadLettered;
+    private readonly Counter<long> _reconnects;
+    private readonly Counter<long> _snapshots;
+    private readonly Func<DateTimeOffset> _utcNow;
+    private bool _disposed;
+
+    public LocalDbMetrics(Func<DateTimeOffset>? utcNow = null)
+    {
+        _utcNow = utcNow ?? (() => DateTimeOffset.UtcNow);
+        _meter = new Meter(MeterName, "0.1.0");
+        _applied = _meter.CreateCounter<long>("localdb.sync.applied");
+        _conflictsDiscarded = _meter.CreateCounter<long>("localdb.sync.conflicts_discarded");
+        _deadLettered = _meter.CreateCounter<long>("localdb.sync.dead_lettered");
+        _reconnects = _meter.CreateCounter<long>("localdb.sync.reconnects");
+        _snapshots = _meter.CreateCounter<long>("localdb.sync.snapshots");
+        _meter.CreateObservableGauge("localdb.oplog.depth", ObserveOplogDepth);
+        _meter.CreateObservableGauge("localdb.sync.lag.seconds", ObserveSyncLag, "s");
+    }
+
+    /// <summary>
+    /// Supplies the current unacked oplog backlog for the <c>localdb.oplog.depth</c> gauge. Invoked
+    /// only at collection time; a cheap synchronous SELECT is acceptable at that frequency.
+    /// </summary>
+    public Func<long>? OplogDepthProvider { get; set; }
+
+    /// <summary>Supplies the last successful sync time for the <c>localdb.sync.lag.seconds</c> gauge; null means no sync yet, and the gauge then reports no measurement.</summary>
+    public Func<DateTimeOffset?>? LastSyncUtcProvider { get; set; }
+
+    /// <summary>The backing meter — exposed so a test <see cref="MeterListener"/> can filter by instance reference rather than the process-shared meter name.</summary>
+    internal Meter Meter => _meter;
+
+    public void RecordApplied(long count)
+    {
+        if (count > 0) _applied.Add(count);
+    }
+
+    public void RecordConflictsDiscarded(long count)
+    {
+        if (count > 0) _conflictsDiscarded.Add(count);
+    }
+
+    public void RecordDeadLettered(long count)
+    {
+        if (count > 0) _deadLettered.Add(count);
+    }
+
+    public void RecordReconnect() => _reconnects.Add(1);
+
+    public void RecordSnapshot(string direction) =>
+        _snapshots.Add(1, new KeyValuePair<string, object?>("direction", direction));
+
+    private IEnumerable<Measurement<long>> ObserveOplogDepth()
+    {
+        var provider = OplogDepthProvider;
+        if (provider is null)
+            return [];
+        try
+        {
+            return [new Measurement<long>(provider())];
+        }
+        catch
+        {
+            // The observation SELECT can race a disposed db / closing connection; a scrape must
+            // never throw, so drop this sample rather than surface a partial value.
+            return [];
+        }
+    }
+
+    private IEnumerable<Measurement<double>> ObserveSyncLag()
+    {
+        var last = LastSyncUtcProvider?.Invoke();
+        if (last is null)
+            return [];
+        var lag = (_utcNow() - last.Value).TotalSeconds;
+        if (lag < 0) lag = 0;
+        return [new Measurement<double>(lag)];
+    }
+
+    public void Dispose()
+    {
+        if (_disposed)
+            return;
+        _meter.Dispose();
+        _disposed = true;
+    }
+}