feat(jetstream): add mirror sync loop and source coordination with filtering (C9+C10)

src/NATS.Server/JetStream/MirrorSource/MirrorCoordinator.cs

@@ -1,22 +1,364 @@
+using System.Threading.Channels;
 using NATS.Server.JetStream.Storage;
 
 namespace NATS.Server.JetStream.MirrorSource;
 
-public sealed class MirrorCoordinator
+// Go reference: server/stream.go:2788-2854 (processMirrorMsgs), 3125-3400 (setupMirrorConsumer)
+// Go reference: server/stream.go:2863-3014 (processInboundMirrorMsg)
+
+/// <summary>
+/// Coordinates continuous synchronization from an origin stream to a local mirror.
+/// Runs a background pull loop that fetches batches of messages from the origin,
+/// applies them to the local store, and tracks origin-to-current sequence alignment
+/// for catchup after restarts. Includes exponential backoff retry on failures
+/// and health reporting via lag calculation.
+/// </summary>
+public sealed class MirrorCoordinator : IAsyncDisposable
 {
+    // Go: sourceHealthCheckInterval = 10 * time.Second
+    private static readonly TimeSpan HealthCheckInterval = TimeSpan.FromSeconds(10);
+
+    // Go: sourceHealthHB = 1 * time.Second
+    private static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(1);
+
+    private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(250);
+    private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(30);
+    private const int DefaultBatchSize = 256;
+
     private readonly IStreamStore _targetStore;
+    private readonly Channel<StoredMessage> _inbound;
+    private readonly Lock _gate = new();
+    private CancellationTokenSource? _cts;
+    private Task? _syncLoop;
+    private int _consecutiveFailures;
+
+    /// <summary>Last sequence number successfully applied from the origin stream.</summary>
     public ulong LastOriginSequence { get; private set; }
+
+    /// <summary>UTC timestamp of the last successful sync operation.</summary>
     public DateTime LastSyncUtc { get; private set; }
 
+    /// <summary>Number of consecutive sync failures (resets on success).</summary>
+    public int ConsecutiveFailures
+    {
+        get { lock (_gate) return _consecutiveFailures; }
+    }
+
+    /// <summary>
+    /// Whether the background sync loop is actively running.
+    /// </summary>
+    public bool IsRunning
+    {
+        get { lock (_gate) return _syncLoop is not null && !_syncLoop.IsCompleted; }
+    }
+
+    /// <summary>
+    /// Current lag: origin last sequence minus local last sequence.
+    /// Returns 0 when fully caught up or when origin sequence is unknown.
+    /// </summary>
+    public ulong Lag { get; private set; }
+
+    // Go: mirror.sseq — stream sequence tracking for gap detection
+    private ulong _expectedOriginSeq;
+
+    // Go: mirror.dseq — delivery sequence tracking
+    private ulong _deliverySeq;
+
     public MirrorCoordinator(IStreamStore targetStore)
     {
         _targetStore = targetStore;
+        _inbound = Channel.CreateUnbounded<StoredMessage>(new UnboundedChannelOptions
+        {
+            SingleReader = true,
+            SingleWriter = false,
+        });
     }
 
+    /// <summary>
+    /// Processes a single inbound message from the origin stream.
+    /// This is the direct-call path used when the origin and mirror are in the same process.
+    /// Go reference: server/stream.go:2863-3014 (processInboundMirrorMsg)
+    /// </summary>
     public async Task OnOriginAppendAsync(StoredMessage message, CancellationToken ct)
     {
+        // Go: sseq == mset.mirror.sseq+1 — normal in-order delivery
+        if (_expectedOriginSeq > 0 && message.Sequence <= _expectedOriginSeq)
+        {
+            // Ignore older/duplicate messages (Go: sseq <= mset.mirror.sseq)
+            return;
+        }
+
+        // Go: sseq > mset.mirror.sseq+1 and dseq == mset.mirror.dseq+1 — gap in origin (deleted/expired)
+        // For in-process mirrors we skip gap handling since the origin store handles its own deletions.
+
         await _targetStore.AppendAsync(message.Subject, message.Payload, ct);
+        _expectedOriginSeq = message.Sequence;
+        _deliverySeq++;
         LastOriginSequence = message.Sequence;
         LastSyncUtc = DateTime.UtcNow;
+        Lag = 0; // In-process mirror receives messages synchronously, so lag is always zero here.
+    }
+
+    /// <summary>
+    /// Enqueues a message for processing by the background sync loop.
+    /// Used when messages arrive asynchronously (e.g., from a pull consumer on the origin).
+    /// </summary>
+    public bool TryEnqueue(StoredMessage message)
+    {
+        return _inbound.Writer.TryWrite(message);
+    }
+
+    /// <summary>
+    /// Starts the background sync loop that drains the inbound channel and applies
+    /// messages to the local store. This models Go's processMirrorMsgs goroutine.
+    /// Go reference: server/stream.go:2788-2854 (processMirrorMsgs)
+    /// </summary>
+    public void StartSyncLoop()
+    {
+        lock (_gate)
+        {
+            if (_syncLoop is not null && !_syncLoop.IsCompleted)
+                return;
+
+            _cts = new CancellationTokenSource();
+            _syncLoop = RunSyncLoopAsync(_cts.Token);
+        }
+    }
+
+    /// <summary>
+    /// Starts the background sync loop with a pull-based fetch from the origin store.
+    /// This models Go's setupMirrorConsumer + processMirrorMsgs pattern where the mirror
+    /// actively pulls batches from the origin.
+    /// Go reference: server/stream.go:3125-3400 (setupMirrorConsumer)
+    /// </summary>
+    public void StartPullSyncLoop(IStreamStore originStore, int batchSize = DefaultBatchSize)
+    {
+        lock (_gate)
+        {
+            if (_syncLoop is not null && !_syncLoop.IsCompleted)
+                return;
+
+            _cts = new CancellationTokenSource();
+            _syncLoop = RunPullSyncLoopAsync(originStore, batchSize, _cts.Token);
+        }
+    }
+
+    /// <summary>
+    /// Stops the background sync loop and waits for it to complete.
+    /// Go reference: server/stream.go:3027-3032 (cancelMirrorConsumer)
+    /// </summary>
+    public async Task StopAsync()
+    {
+        CancellationTokenSource? cts;
+        Task? loop;
+        lock (_gate)
+        {
+            cts = _cts;
+            loop = _syncLoop;
+        }
+
+        if (cts is not null)
+        {
+            await cts.CancelAsync();
+            if (loop is not null)
+            {
+                try { await loop; }
+                catch (OperationCanceledException) { }
+            }
+        }
+
+        lock (_gate)
+        {
+            _cts?.Dispose();
+            _cts = null;
+            _syncLoop = null;
+        }
+    }
+
+    /// <summary>
+    /// Reports current health state for monitoring.
+    /// Go reference: server/stream.go:2739-2743 (mirrorInfo), 2698-2736 (sourceInfo)
+    /// </summary>
+    public MirrorHealthReport GetHealthReport(ulong? originLastSeq = null)
+    {
+        var lag = originLastSeq.HasValue && originLastSeq.Value > LastOriginSequence
+            ? originLastSeq.Value - LastOriginSequence
+            : Lag;
+
+        return new MirrorHealthReport
+        {
+            LastOriginSequence = LastOriginSequence,
+            LastSyncUtc = LastSyncUtc,
+            Lag = lag,
+            ConsecutiveFailures = ConsecutiveFailures,
+            IsRunning = IsRunning,
+            IsStalled = LastSyncUtc != default
+                        && DateTime.UtcNow - LastSyncUtc > HealthCheckInterval,
+        };
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await StopAsync();
+        _inbound.Writer.TryComplete();
+    }
+
+    // -------------------------------------------------------------------------
+    // Background sync loop: channel-based (inbound messages pushed to us)
+    // Go reference: server/stream.go:2788-2854 (processMirrorMsgs main loop)
+    // -------------------------------------------------------------------------
+    private async Task RunSyncLoopAsync(CancellationToken ct)
+    {
+        // Go: t := time.NewTicker(sourceHealthCheckInterval)
+        using var healthTimer = new PeriodicTimer(HealthCheckInterval);
+        var reader = _inbound.Reader;
+
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                // Go: select { case <-msgs.ch: ... case <-t.C: ... }
+                // We process all available messages, then wait for more or health check.
+                while (reader.TryRead(out var msg))
+                {
+                    await ProcessInboundMessageAsync(msg, ct);
+                }
+
+                // Wait for either a new message or health check tick
+                var readTask = reader.WaitToReadAsync(ct).AsTask();
+                var healthTask = healthTimer.WaitForNextTickAsync(ct).AsTask();
+                await Task.WhenAny(readTask, healthTask);
+
+                if (ct.IsCancellationRequested)
+                    break;
+
+                // Drain any messages that arrived
+                while (reader.TryRead(out var msg2))
+                {
+                    await ProcessInboundMessageAsync(msg2, ct);
+                }
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception)
+            {
+                // Go: mset.retryMirrorConsumer() on errors
+                lock (_gate)
+                {
+                    _consecutiveFailures++;
+                }
+
+                var delay = CalculateBackoff(_consecutiveFailures);
+                try { await Task.Delay(delay, ct); }
+                catch (OperationCanceledException) { break; }
+            }
+        }
+    }
+
+    // -------------------------------------------------------------------------
+    // Background sync loop: pull-based (we fetch from origin)
+    // Go reference: server/stream.go:3125-3400 (setupMirrorConsumer creates
+    //               ephemeral pull consumer; processMirrorMsgs drains it)
+    // -------------------------------------------------------------------------
+    private async Task RunPullSyncLoopAsync(IStreamStore originStore, int batchSize, CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                var messages = await originStore.ListAsync(ct);
+                var applied = 0;
+
+                foreach (var msg in messages)
+                {
+                    if (ct.IsCancellationRequested) break;
+
+                    // Skip messages we've already synced
+                    if (msg.Sequence <= LastOriginSequence)
+                        continue;
+
+                    await ProcessInboundMessageAsync(msg, ct);
+                    applied++;
+
+                    if (applied >= batchSize)
+                        break;
+                }
+
+                // Update lag based on origin state
+                if (messages.Count > 0)
+                {
+                    var originLast = messages[^1].Sequence;
+                    Lag = originLast > LastOriginSequence ? originLast - LastOriginSequence : 0;
+                }
+
+                lock (_gate) _consecutiveFailures = 0;
+
+                // Go: If caught up, wait briefly before next poll
+                if (applied == 0)
+                {
+                    try { await Task.Delay(HeartbeatInterval, ct); }
+                    catch (OperationCanceledException) { break; }
+                }
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception)
+            {
+                lock (_gate) _consecutiveFailures++;
+                var delay = CalculateBackoff(_consecutiveFailures);
+                try { await Task.Delay(delay, ct); }
+                catch (OperationCanceledException) { break; }
+            }
+        }
+    }
+
+    // Go reference: server/stream.go:2863-3014 (processInboundMirrorMsg)
+    private async Task ProcessInboundMessageAsync(StoredMessage message, CancellationToken ct)
+    {
+        // Go: sseq <= mset.mirror.sseq — ignore older messages
+        if (_expectedOriginSeq > 0 && message.Sequence <= _expectedOriginSeq)
+            return;
+
+        // Go: dc > 1 — skip redelivered messages
+        if (message.Redelivered)
+            return;
+
+        // Go: sseq == mset.mirror.sseq+1 — normal sequential delivery
+        // Go: else — gap handling (skip sequences if deliver seq matches)
+        await _targetStore.AppendAsync(message.Subject, message.Payload, ct);
+        _expectedOriginSeq = message.Sequence;
+        _deliverySeq++;
+        LastOriginSequence = message.Sequence;
+        LastSyncUtc = DateTime.UtcNow;
+
+        lock (_gate) _consecutiveFailures = 0;
+    }
+
+    // Go reference: server/stream.go:3478-3505 (calculateRetryBackoff in setupSourceConsumer)
+    // Exponential backoff with jitter, capped at MaxRetryDelay.
+    private static TimeSpan CalculateBackoff(int failures)
+    {
+        var baseDelay = InitialRetryDelay.TotalMilliseconds * Math.Pow(2, Math.Min(failures - 1, 10));
+        var capped = Math.Min(baseDelay, MaxRetryDelay.TotalMilliseconds);
+        var jitter = Random.Shared.NextDouble() * 0.2 * capped; // +-20% jitter
+        return TimeSpan.FromMilliseconds(capped + jitter);
     }
 }
+
+/// <summary>
+/// Health report for a mirror coordinator, used by monitoring endpoints.
+/// Go reference: server/stream.go:2698-2736 (sourceInfo/StreamSourceInfo)
+/// </summary>
+public sealed record MirrorHealthReport
+{
+    public ulong LastOriginSequence { get; init; }
+    public DateTime LastSyncUtc { get; init; }
+    public ulong Lag { get; init; }
+    public int ConsecutiveFailures { get; init; }
+    public bool IsRunning { get; init; }
+    public bool IsStalled { get; init; }
+}

src/NATS.Server/JetStream/MirrorSource/SourceCoordinator.cs

@@ -1,23 +1,109 @@
-using NATS.Server.JetStream.Storage;
+using System.Collections.Concurrent;
+using System.Threading.Channels;
 using NATS.Server.JetStream.Models;
+using NATS.Server.JetStream.Storage;
+using NATS.Server.Subscriptions;
 
 namespace NATS.Server.JetStream.MirrorSource;
 
-public sealed class SourceCoordinator
+// Go reference: server/stream.go:3860-4007 (processInboundSourceMsg)
+// Go reference: server/stream.go:3752-3833 (processAllSourceMsgs)
+// Go reference: server/stream.go:3474-3720 (setupSourceConsumer, trySetupSourceConsumer)
+
+/// <summary>
+/// Coordinates consumption from a source stream into a target stream with support for:
+/// - Subject filtering via FilterSubject (Go: StreamSource.FilterSubject)
+/// - Subject transform prefix applied before storing (Go: SubjectTransforms)
+/// - Account isolation via SourceAccount
+/// - Deduplication via Nats-Msg-Id header with configurable window
+/// - Lag tracking per source
+/// - Background sync loop with exponential backoff retry
+/// </summary>
+public sealed class SourceCoordinator : IAsyncDisposable
 {
+    // Go: sourceHealthCheckInterval = 10 * time.Second
+    private static readonly TimeSpan HealthCheckInterval = TimeSpan.FromSeconds(10);
+
+    // Go: sourceHealthHB = 1 * time.Second
+    private static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(1);
+
+    private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(250);
+    private static readonly TimeSpan MaxRetryDelay = TimeSpan.FromSeconds(30);
+    private const int DefaultBatchSize = 256;
+
     private readonly IStreamStore _targetStore;
     private readonly StreamSourceConfig _sourceConfig;
+    private readonly Channel<StoredMessage> _inbound;
+    private readonly Lock _gate = new();
+    private CancellationTokenSource? _cts;
+    private Task? _syncLoop;
+    private int _consecutiveFailures;
+
+    // Go: si.sseq — last stream sequence from origin
+    private ulong _expectedOriginSeq;
+
+    // Go: si.dseq — delivery sequence tracking
+    private ulong _deliverySeq;
+
+    // Deduplication state: tracks recently seen Nats-Msg-Id values with their timestamps.
+    // Go: server/stream.go doesn't have per-source dedup, but the stream's duplicate window
+    // (DuplicateWindowMs) applies to publishes. We implement source-level dedup here.
+    private readonly ConcurrentDictionary<string, DateTime> _dedupWindow = new(StringComparer.Ordinal);
+    private DateTime _lastDedupPrune = DateTime.UtcNow;
+
+    /// <summary>Last sequence number successfully applied from the origin stream.</summary>
     public ulong LastOriginSequence { get; private set; }
+
+    /// <summary>UTC timestamp of the last successful sync operation.</summary>
     public DateTime LastSyncUtc { get; private set; }
 
+    /// <summary>Number of consecutive sync failures (resets on success).</summary>
+    public int ConsecutiveFailures
+    {
+        get { lock (_gate) return _consecutiveFailures; }
+    }
+
+    /// <summary>Whether the background sync loop is actively running.</summary>
+    public bool IsRunning
+    {
+        get { lock (_gate) return _syncLoop is not null && !_syncLoop.IsCompleted; }
+    }
+
+    /// <summary>
+    /// Current lag: origin last sequence minus local last sequence.
+    /// Returns 0 when fully caught up.
+    /// </summary>
+    public ulong Lag { get; private set; }
+
+    /// <summary>Total messages dropped by the subject filter.</summary>
+    public long FilteredOutCount { get; private set; }
+
+    /// <summary>Total messages dropped by deduplication.</summary>
+    public long DeduplicatedCount { get; private set; }
+
+    /// <summary>The source configuration driving this coordinator.</summary>
+    public StreamSourceConfig Config => _sourceConfig;
+
     public SourceCoordinator(IStreamStore targetStore, StreamSourceConfig sourceConfig)
     {
         _targetStore = targetStore;
         _sourceConfig = sourceConfig;
+        _inbound = Channel.CreateUnbounded<StoredMessage>(new UnboundedChannelOptions
+        {
+            SingleReader = true,
+            SingleWriter = false,
+        });
     }
 
+    /// <summary>
+    /// Processes a single inbound message from the origin stream.
+    /// This is the direct-call path used when the origin and target are in the same process.
+    /// Go reference: server/stream.go:3860-4007 (processInboundSourceMsg)
+    /// </summary>
     public async Task OnOriginAppendAsync(StoredMessage message, CancellationToken ct)
     {
+        // Account isolation: skip messages from different accounts.
+        // Go: This is checked at the subscription level, but we enforce it here for in-process sources.
         if (!string.IsNullOrWhiteSpace(_sourceConfig.SourceAccount)
             && !string.IsNullOrWhiteSpace(message.Account)
             && !string.Equals(_sourceConfig.SourceAccount, message.Account, StringComparison.Ordinal))
@@ -25,12 +111,360 @@ public sealed class SourceCoordinator
             return;
         }
 
+        // Subject filter: only forward messages matching the filter.
+        // Go: server/stream.go:3597-3598 — if ssi.FilterSubject != _EMPTY_ { req.Config.FilterSubject = ssi.FilterSubject }
+        if (!string.IsNullOrWhiteSpace(_sourceConfig.FilterSubject)
+            && !SubjectMatch.MatchLiteral(message.Subject, _sourceConfig.FilterSubject))
+        {
+            FilteredOutCount++;
+            return;
+        }
+
+        // Deduplication: check Nats-Msg-Id header against the dedup window.
+        if (_sourceConfig.DuplicateWindowMs > 0 && message.MsgId is not null)
+        {
+            if (IsDuplicate(message.MsgId))
+            {
+                DeduplicatedCount++;
+                return;
+            }
+
+            RecordMsgId(message.MsgId);
+        }
+
+        // Go: si.sseq <= current — ignore older/duplicate messages
+        if (_expectedOriginSeq > 0 && message.Sequence <= _expectedOriginSeq)
+            return;
+
+        // Subject transform: apply prefix before storing.
+        // Go: server/stream.go:3943-3956 (subject transform for the source)
         var subject = message.Subject;
         if (!string.IsNullOrWhiteSpace(_sourceConfig.SubjectTransformPrefix))
             subject = $"{_sourceConfig.SubjectTransformPrefix}{subject}";
 
         await _targetStore.AppendAsync(subject, message.Payload, ct);
+        _expectedOriginSeq = message.Sequence;
+        _deliverySeq++;
         LastOriginSequence = message.Sequence;
         LastSyncUtc = DateTime.UtcNow;
+        Lag = 0;
+    }
+
+    /// <summary>
+    /// Enqueues a message for processing by the background sync loop.
+    /// </summary>
+    public bool TryEnqueue(StoredMessage message)
+    {
+        return _inbound.Writer.TryWrite(message);
+    }
+
+    /// <summary>
+    /// Starts the background sync loop that drains the inbound channel.
+    /// Go reference: server/stream.go:3752-3833 (processAllSourceMsgs)
+    /// </summary>
+    public void StartSyncLoop()
+    {
+        lock (_gate)
+        {
+            if (_syncLoop is not null && !_syncLoop.IsCompleted)
+                return;
+
+            _cts = new CancellationTokenSource();
+            _syncLoop = RunSyncLoopAsync(_cts.Token);
+        }
+    }
+
+    /// <summary>
+    /// Starts a pull-based sync loop that actively fetches from the origin store.
+    /// Go reference: server/stream.go:3474-3720 (setupSourceConsumer + trySetupSourceConsumer)
+    /// </summary>
+    public void StartPullSyncLoop(IStreamStore originStore, int batchSize = DefaultBatchSize)
+    {
+        lock (_gate)
+        {
+            if (_syncLoop is not null && !_syncLoop.IsCompleted)
+                return;
+
+            _cts = new CancellationTokenSource();
+            _syncLoop = RunPullSyncLoopAsync(originStore, batchSize, _cts.Token);
+        }
+    }
+
+    /// <summary>
+    /// Stops the background sync loop.
+    /// Go reference: server/stream.go:3438-3469 (cancelSourceConsumer)
+    /// </summary>
+    public async Task StopAsync()
+    {
+        CancellationTokenSource? cts;
+        Task? loop;
+        lock (_gate)
+        {
+            cts = _cts;
+            loop = _syncLoop;
+        }
+
+        if (cts is not null)
+        {
+            await cts.CancelAsync();
+            if (loop is not null)
+            {
+                try { await loop; }
+                catch (OperationCanceledException) { }
+            }
+        }
+
+        lock (_gate)
+        {
+            _cts?.Dispose();
+            _cts = null;
+            _syncLoop = null;
+        }
+    }
+
+    /// <summary>
+    /// Reports current health state for monitoring.
+    /// Go reference: server/stream.go:2687-2695 (sourcesInfo)
+    /// </summary>
+    public SourceHealthReport GetHealthReport(ulong? originLastSeq = null)
+    {
+        var lag = originLastSeq.HasValue && originLastSeq.Value > LastOriginSequence
+            ? originLastSeq.Value - LastOriginSequence
+            : Lag;
+
+        return new SourceHealthReport
+        {
+            SourceName = _sourceConfig.Name,
+            FilterSubject = _sourceConfig.FilterSubject,
+            LastOriginSequence = LastOriginSequence,
+            LastSyncUtc = LastSyncUtc,
+            Lag = lag,
+            ConsecutiveFailures = ConsecutiveFailures,
+            IsRunning = IsRunning,
+            IsStalled = LastSyncUtc != default
+                        && DateTime.UtcNow - LastSyncUtc > HealthCheckInterval,
+            FilteredOutCount = FilteredOutCount,
+            DeduplicatedCount = DeduplicatedCount,
+        };
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await StopAsync();
+        _inbound.Writer.TryComplete();
+    }
+
+    // -------------------------------------------------------------------------
+    // Background sync loop: channel-based
+    // Go reference: server/stream.go:3752-3833 (processAllSourceMsgs)
+    // -------------------------------------------------------------------------
+    private async Task RunSyncLoopAsync(CancellationToken ct)
+    {
+        using var healthTimer = new PeriodicTimer(HealthCheckInterval);
+        var reader = _inbound.Reader;
+
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                while (reader.TryRead(out var msg))
+                {
+                    await ProcessInboundMessageAsync(msg, ct);
+                }
+
+                var readTask = reader.WaitToReadAsync(ct).AsTask();
+                var healthTask = healthTimer.WaitForNextTickAsync(ct).AsTask();
+                await Task.WhenAny(readTask, healthTask);
+
+                if (ct.IsCancellationRequested)
+                    break;
+
+                while (reader.TryRead(out var msg2))
+                {
+                    await ProcessInboundMessageAsync(msg2, ct);
+                }
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception)
+            {
+                lock (_gate) _consecutiveFailures++;
+                var delay = CalculateBackoff(_consecutiveFailures);
+                try { await Task.Delay(delay, ct); }
+                catch (OperationCanceledException) { break; }
+            }
+        }
+    }
+
+    // -------------------------------------------------------------------------
+    // Background sync loop: pull-based
+    // Go reference: server/stream.go:3474-3720 (setupSourceConsumer)
+    // -------------------------------------------------------------------------
+    private async Task RunPullSyncLoopAsync(IStreamStore originStore, int batchSize, CancellationToken ct)
+    {
+        while (!ct.IsCancellationRequested)
+        {
+            try
+            {
+                var messages = await originStore.ListAsync(ct);
+                var applied = 0;
+
+                foreach (var msg in messages)
+                {
+                    if (ct.IsCancellationRequested) break;
+
+                    if (msg.Sequence <= LastOriginSequence)
+                        continue;
+
+                    await ProcessInboundMessageAsync(msg, ct);
+                    applied++;
+
+                    if (applied >= batchSize)
+                        break;
+                }
+
+                // Update lag
+                if (messages.Count > 0)
+                {
+                    var originLast = messages[^1].Sequence;
+                    Lag = originLast > LastOriginSequence ? originLast - LastOriginSequence : 0;
+                }
+
+                lock (_gate) _consecutiveFailures = 0;
+
+                if (applied == 0)
+                {
+                    try { await Task.Delay(HeartbeatInterval, ct); }
+                    catch (OperationCanceledException) { break; }
+                }
+            }
+            catch (OperationCanceledException) when (ct.IsCancellationRequested)
+            {
+                break;
+            }
+            catch (Exception)
+            {
+                lock (_gate) _consecutiveFailures++;
+                var delay = CalculateBackoff(_consecutiveFailures);
+                try { await Task.Delay(delay, ct); }
+                catch (OperationCanceledException) { break; }
+            }
+        }
+    }
+
+    // Go reference: server/stream.go:3860-4007 (processInboundSourceMsg)
+    private async Task ProcessInboundMessageAsync(StoredMessage message, CancellationToken ct)
+    {
+        // Account isolation
+        if (!string.IsNullOrWhiteSpace(_sourceConfig.SourceAccount)
+            && !string.IsNullOrWhiteSpace(message.Account)
+            && !string.Equals(_sourceConfig.SourceAccount, message.Account, StringComparison.Ordinal))
+        {
+            return;
+        }
+
+        // Subject filter
+        if (!string.IsNullOrWhiteSpace(_sourceConfig.FilterSubject)
+            && !SubjectMatch.MatchLiteral(message.Subject, _sourceConfig.FilterSubject))
+        {
+            FilteredOutCount++;
+            return;
+        }
+
+        // Deduplication
+        if (_sourceConfig.DuplicateWindowMs > 0 && message.MsgId is not null)
+        {
+            if (IsDuplicate(message.MsgId))
+            {
+                DeduplicatedCount++;
+                return;
+            }
+
+            RecordMsgId(message.MsgId);
+        }
+
+        // Skip already-seen sequences
+        if (_expectedOriginSeq > 0 && message.Sequence <= _expectedOriginSeq)
+            return;
+
+        // Redelivery check (Go: dc > 1)
+        if (message.Redelivered)
+            return;
+
+        // Subject transform
+        var subject = message.Subject;
+        if (!string.IsNullOrWhiteSpace(_sourceConfig.SubjectTransformPrefix))
+            subject = $"{_sourceConfig.SubjectTransformPrefix}{subject}";
+
+        await _targetStore.AppendAsync(subject, message.Payload, ct);
+        _expectedOriginSeq = message.Sequence;
+        _deliverySeq++;
+        LastOriginSequence = message.Sequence;
+        LastSyncUtc = DateTime.UtcNow;
+
+        lock (_gate) _consecutiveFailures = 0;
+    }
+
+    // -------------------------------------------------------------------------
+    // Deduplication helpers
+    // -------------------------------------------------------------------------
+
+    private bool IsDuplicate(string msgId)
+    {
+        PruneDedupWindowIfNeeded();
+        return _dedupWindow.ContainsKey(msgId);
+    }
+
+    private void RecordMsgId(string msgId)
+    {
+        _dedupWindow[msgId] = DateTime.UtcNow;
+    }
+
+    private void PruneDedupWindowIfNeeded()
+    {
+        if (_sourceConfig.DuplicateWindowMs <= 0)
+            return;
+
+        var now = DateTime.UtcNow;
+        // Prune at most once per second to avoid overhead
+        if ((now - _lastDedupPrune).TotalMilliseconds < 1000)
+            return;
+
+        _lastDedupPrune = now;
+        var cutoff = now.AddMilliseconds(-_sourceConfig.DuplicateWindowMs);
+        foreach (var kvp in _dedupWindow)
+        {
+            if (kvp.Value < cutoff)
+                _dedupWindow.TryRemove(kvp.Key, out _);
+        }
+    }
+
+    // Go reference: server/stream.go:3478-3505 (calculateRetryBackoff)
+    private static TimeSpan CalculateBackoff(int failures)
+    {
+        var baseDelay = InitialRetryDelay.TotalMilliseconds * Math.Pow(2, Math.Min(failures - 1, 10));
+        var capped = Math.Min(baseDelay, MaxRetryDelay.TotalMilliseconds);
+        var jitter = Random.Shared.NextDouble() * 0.2 * capped;
+        return TimeSpan.FromMilliseconds(capped + jitter);
     }
 }
+
+/// <summary>
+/// Health report for a source coordinator, used by monitoring endpoints.
+/// Go reference: server/stream.go:2687-2736 (sourcesInfo, sourceInfo)
+/// </summary>
+public sealed record SourceHealthReport
+{
+    public string SourceName { get; init; } = string.Empty;
+    public string? FilterSubject { get; init; }
+    public ulong LastOriginSequence { get; init; }
+    public DateTime LastSyncUtc { get; init; }
+    public ulong Lag { get; init; }
+    public int ConsecutiveFailures { get; init; }
+    public bool IsRunning { get; init; }
+    public bool IsStalled { get; init; }
+    public long FilteredOutCount { get; init; }
+    public long DeduplicatedCount { get; init; }
+}

src/NATS.Server/JetStream/Models/StreamConfig.cs

@@ -35,4 +35,12 @@ public sealed class StreamSourceConfig
     public string Name { get; set; } = string.Empty;
     public string? SubjectTransformPrefix { get; set; }
     public string? SourceAccount { get; set; }
+
+    // Go: StreamSource.FilterSubject — only forward messages matching this subject filter.
+    public string? FilterSubject { get; set; }
+
+    // Deduplication window in milliseconds for Nats-Msg-Id header-based dedup.
+    // Defaults to 0 (disabled). When > 0, duplicate messages with the same Nats-Msg-Id
+    // within this window are silently dropped.
+    public int DuplicateWindowMs { get; set; }
 }

src/NATS.Server/JetStream/Storage/StoredMessage.cs

@@ -8,4 +8,14 @@ public sealed class StoredMessage
     public DateTime TimestampUtc { get; init; } = DateTime.UtcNow;
     public string? Account { get; init; }
     public bool Redelivered { get; init; }
+
+    /// <summary>
+    /// Optional message headers. Used for deduplication (Nats-Msg-Id) and source tracking.
+    /// </summary>
+    public IReadOnlyDictionary<string, string>? Headers { get; init; }
+
+    /// <summary>
+    /// Convenience accessor for the Nats-Msg-Id header value, used by source deduplication.
+    /// </summary>
+    public string? MsgId => Headers is not null && Headers.TryGetValue("Nats-Msg-Id", out var id) ? id : null;
 }

src/NATS.Server/JetStream/StreamManager.cs

@@ -336,6 +336,8 @@ public sealed class StreamManager
                 Name = s.Name,
                 SubjectTransformPrefix = s.SubjectTransformPrefix,
                 SourceAccount = s.SourceAccount,
+                FilterSubject = s.FilterSubject,
+                DuplicateWindowMs = s.DuplicateWindowMs,
             })],
         };