feat(mirror): add exponential backoff retry, gap detection, and error tracking

Exposes public RecordFailure/RecordSuccess/GetRetryDelay (no-jitter, deterministic)
on MirrorCoordinator, plus RecordSourceSeq with HasGap/GapStart/GapEnd properties
and SetError/ClearError/HasError/ErrorMessage for error state. Makes IsDuplicate
and RecordMsgId public on SourceCoordinator and adds PruneDedupWindow(DateTimeOffset)
for explicit-cutoff dedup window pruning. Adds 5 unit tests in MirrorSourceRetryTests.
This commit is contained in:
Joseph Doherty
2026-02-25 02:30:55 -05:00
parent 955d568423
commit 8fa16d59d2
3 changed files with 236 additions and 2 deletions

View File

@@ -58,12 +58,44 @@ public sealed class MirrorCoordinator : IAsyncDisposable
/// </summary>
public ulong Lag { get; private set; }
// -------------------------------------------------------------------------
// Gap detection properties
// Go reference: server/stream.go:2863-3014 (processInboundMirrorMsg gap handling)
// -------------------------------------------------------------------------
/// <summary>Whether a sequence gap has been detected in the origin stream.</summary>
public bool HasGap => _hasGap;
/// <summary>First missing sequence number when a gap is detected.</summary>
public ulong GapStart => _gapStart;
/// <summary>Last missing sequence number when a gap is detected.</summary>
public ulong GapEnd => _gapEnd;
// -------------------------------------------------------------------------
// Error state properties
// -------------------------------------------------------------------------
/// <summary>Whether the coordinator is in an error state.</summary>
public bool HasError => _errorMessage is not null;
/// <summary>Current error message, or null if no error.</summary>
public string? ErrorMessage => _errorMessage;
// Go: mirror.sseq — stream sequence tracking for gap detection
private ulong _expectedOriginSeq;
// Go: mirror.dseq — delivery sequence tracking
private ulong _deliverySeq;
// Gap detection state
private ulong _gapStart;
private ulong _gapEnd;
private bool _hasGap;
// Error state tracking
private string? _errorMessage;
public MirrorCoordinator(IStreamStore targetStore)
{
_targetStore = targetStore;
@@ -175,6 +207,74 @@ public sealed class MirrorCoordinator : IAsyncDisposable
}
}
// -------------------------------------------------------------------------
// Public retry / failure tracking API
// Go reference: server/stream.go:3478-3505 (calculateRetryBackoff),
// server/stream.go:3125-3400 (setupMirrorConsumer retry logic)
// -------------------------------------------------------------------------
/// <summary>
/// Increments the consecutive failure counter.
/// Call this when a sync attempt fails.
/// </summary>
public void RecordFailure()
{
lock (_gate) _consecutiveFailures++;
}
/// <summary>
/// Resets the consecutive failure counter to zero.
/// Call this when a sync attempt succeeds.
/// </summary>
public void RecordSuccess()
{
lock (_gate) _consecutiveFailures = 0;
}
/// <summary>
/// Returns the exponential backoff delay for the current failure count,
/// without jitter so that tests get deterministic results.
/// Go reference: server/stream.go:3478-3505 (calculateRetryBackoff)
/// </summary>
public TimeSpan GetRetryDelay()
{
var failures = ConsecutiveFailures;
if (failures == 0) return InitialRetryDelay;
var baseDelay = InitialRetryDelay.TotalMilliseconds * Math.Pow(2, Math.Min(failures - 1, 10));
var capped = Math.Min(baseDelay, MaxRetryDelay.TotalMilliseconds);
return TimeSpan.FromMilliseconds(capped);
}
// -------------------------------------------------------------------------
// Sequence gap detection
// Go reference: server/stream.go:2863-3014 (processInboundMirrorMsg)
// -------------------------------------------------------------------------
/// <summary>
/// Records the next received sequence number from the origin stream.
/// Sets gap state when a gap (skipped sequences) is detected.
/// </summary>
public void RecordSourceSeq(ulong seq)
{
if (_expectedOriginSeq > 0 && seq > _expectedOriginSeq + 1)
{
_hasGap = true;
_gapStart = _expectedOriginSeq + 1;
_gapEnd = seq - 1;
}
_expectedOriginSeq = seq;
}
// -------------------------------------------------------------------------
// Error state management
// -------------------------------------------------------------------------
/// <summary>Sets the coordinator into an error state with the given message.</summary>
public void SetError(string message) => _errorMessage = message;
/// <summary>Clears the error state.</summary>
public void ClearError() => _errorMessage = null;
/// <summary>
/// Reports current health state for monitoring.
/// Go reference: server/stream.go:2739-2743 (mirrorInfo), 2698-2736 (sourceInfo)

View File

@@ -411,17 +411,41 @@ public sealed class SourceCoordinator : IAsyncDisposable
// Deduplication helpers
// -------------------------------------------------------------------------
private bool IsDuplicate(string msgId)
/// <summary>
/// Returns true if the given message ID is already present in the dedup window.
/// Go reference: server/stream.go duplicate window check
/// </summary>
public bool IsDuplicate(string msgId)
{
PruneDedupWindowIfNeeded();
return _dedupWindow.ContainsKey(msgId);
}
private void RecordMsgId(string msgId)
/// <summary>
/// Records a message ID in the dedup window with the current timestamp.
/// Go reference: server/stream.go duplicate window tracking
/// </summary>
public void RecordMsgId(string msgId)
{
_dedupWindow[msgId] = DateTime.UtcNow;
}
/// <summary>
/// Removes all dedup window entries whose timestamps are earlier than the given cutoff.
/// This is the explicit-cutoff variant intended for testing, as opposed to the internal
/// time-based pruning done by <see cref="PruneDedupWindowIfNeeded"/>.
/// Go reference: server/stream.go duplicate window pruning
/// </summary>
public void PruneDedupWindow(DateTimeOffset cutoff)
{
var cutoffDt = cutoff.UtcDateTime;
foreach (var kvp in _dedupWindow)
{
if (kvp.Value < cutoffDt)
_dedupWindow.TryRemove(kvp.Key, out _);
}
}
private void PruneDedupWindowIfNeeded()
{
if (_sourceConfig.DuplicateWindowMs <= 0)