feat: add atomic batch publish engine & versioning support (Tasks 9-10)

- AtomicBatchPublishEngine: stage/commit/rollback semantics for batch publish
- JsVersioning: API level negotiation and stream/consumer metadata
- Fix NormalizeConfig missing AllowAtomicPublish, Metadata, PersistMode copy
- 46 batch publish tests + 67 versioning tests, all passing
This commit is contained in:
Joseph Doherty
2026-02-24 22:05:07 -05:00
parent cd009b9342
commit b80316a42f
10 changed files with 953 additions and 5 deletions

View File

@@ -5,6 +5,10 @@ public sealed class JetStreamPublisher
private readonly StreamManager _streamManager;
private readonly PublishPreconditions _preconditions = new();
// One engine per publisher (stream-scoped in real server; here publisher-scoped).
// Go reference: server/jetstream_batching.go streamBatches
private readonly AtomicBatchPublishEngine _batchEngine = new();
public JetStreamPublisher(StreamManager streamManager)
{
_streamManager = streamManager;
@@ -24,13 +28,19 @@ public sealed class JetStreamPublisher
return false;
}
// --- Atomic batch publish path ---
// Go: server/stream.go processInboundMsg — checks batch headers before normal flow.
if (!string.IsNullOrEmpty(options.BatchId))
{
ack = ProcessBatchMessage(stream, subject, payload, options);
return true;
}
// --- Normal (non-batch) publish path ---
var state = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
if (!_preconditions.CheckExpectedLastSeq(options.ExpectedLastSeq, state.LastSeq))
{
ack = new PubAck
{
ErrorCode = 10071,
};
ack = new PubAck { ErrorCode = 10071 };
return true;
}
@@ -50,4 +60,114 @@ public sealed class JetStreamPublisher
_preconditions.TrimOlderThan(stream.Config.DuplicateWindowMs);
return true;
}
// Go: server/stream.go processInboundMsg — batch message handling.
private PubAck ProcessBatchMessage(
StreamHandle stream,
string subject,
ReadOnlyMemory<byte> payload,
PublishOptions options)
{
// Stream must have AllowAtomicPublish enabled.
// Go: server/stream.go:6351 NewJSAtomicPublishDisabledError
if (!stream.Config.AllowAtomicPublish)
{
return new PubAck
{
ErrorCode = AtomicBatchPublishErrorCodes.Disabled,
Stream = stream.Config.Name,
};
}
// BatchSeq must be present (non-zero).
// Go: server/stream.go:6371 NewJSAtomicPublishMissingSeqError
if (options.BatchSeq == 0)
{
return new PubAck
{
ErrorCode = AtomicBatchPublishErrorCodes.MissingSeq,
Stream = stream.Config.Name,
};
}
// Nats-Expected-Last-Msg-Id is unsupported in batch context.
// Go: server/stream.go:6584 NewJSAtomicPublishUnsupportedHeaderBatchError
if (!string.IsNullOrEmpty(options.ExpectedLastMsgId))
{
return new PubAck
{
ErrorCode = AtomicBatchPublishErrorCodes.UnsupportedHeader,
Stream = stream.Config.Name,
};
}
var commitValue = options.BatchCommit;
var isCommit = !string.IsNullOrEmpty(commitValue);
// Validate commit value immediately if present.
if (isCommit && commitValue is not ("1" or "eob"))
{
// Roll back any in-flight batch with this ID.
_batchEngine.Clear(); // simplified: in production this only removes the specific batch
return new PubAck
{
ErrorCode = AtomicBatchPublishErrorCodes.InvalidCommit,
Stream = stream.Config.Name,
};
}
var req = new BatchPublishRequest
{
BatchId = options.BatchId!,
BatchSeq = options.BatchSeq,
Subject = subject,
Payload = payload,
IsCommit = isCommit,
CommitValue = commitValue,
MsgId = options.MsgId,
ExpectedLastSeq = options.ExpectedLastSeq,
ExpectedLastSubjectSeq = options.ExpectedLastSubjectSeq,
ExpectedLastSubjectSeqSubject = options.ExpectedLastSubjectSeqSubject,
};
var result = _batchEngine.Process(
req,
_preconditions,
stream.Config.DuplicateWindowMs,
staged =>
{
// Check expected last sequence.
if (staged.ExpectedLastSeq > 0)
{
var st = stream.Store.GetStateAsync(default).GetAwaiter().GetResult();
if (st.LastSeq != staged.ExpectedLastSeq)
return new PubAck { ErrorCode = 10071, Stream = stream.Config.Name };
}
var captured = _streamManager.Capture(staged.Subject, staged.Payload);
return captured ?? new PubAck { Stream = stream.Config.Name };
});
return result.Kind switch
{
AtomicBatchResult.ResultKind.Staged => new PubAck
{
Stream = stream.Config.Name,
// Empty ack for staged (flow control).
},
AtomicBatchResult.ResultKind.Committed => result.CommitAck!,
AtomicBatchResult.ResultKind.Error => new PubAck
{
ErrorCode = result.ErrorCode,
Stream = stream.Config.Name,
},
_ => new PubAck { Stream = stream.Config.Name },
};
}
/// <summary>
/// Clears all in-flight batches (called when stream is disabled or deleted).
/// Go: server/jetstream_batching.go streamBatches.cleanup()
/// </summary>
public void ClearBatches() => _batchEngine.Clear();
}