feat: add atomic batch publish engine & versioning support (Tasks 9-10)
- AtomicBatchPublishEngine: stage/commit/rollback semantics for batch publish - JsVersioning: API level negotiation and stream/consumer metadata - Fix NormalizeConfig missing AllowAtomicPublish, Metadata, PersistMode copy - 46 batch publish tests + 67 versioning tests, all passing
This commit is contained in:
Joseph Doherty
299
src/NATS.Server/JetStream/JsVersioning.cs
Normal file
299
src/NATS.Server/JetStream/JsVersioning.cs
Normal file
@@ -0,0 +1,299 @@
|
||||
// Go reference: golang/nats-server/server/jetstream_versioning.go
|
||||
// Versioning metadata management for JetStream streams and consumers.
|
||||
// Manages the _nats.req.level, _nats.ver, _nats.level metadata keys.
|
||||
|
||||
using NATS.Server.JetStream.Models;
|
||||
|
||||
namespace NATS.Server.JetStream;
|
||||
|
||||
/// <summary>
|
||||
/// JetStream API versioning constants and metadata management.
|
||||
/// Go reference: server/jetstream_versioning.go
|
||||
/// </summary>
|
||||
public static class JsVersioning
|
||||
{
|
||||
/// <summary>
|
||||
/// Current JetStream API level supported by this server.
|
||||
/// Go: JSApiLevel = 3 (jetstream_versioning.go:20)
|
||||
/// </summary>
|
||||
public const int JsApiLevel = 3;
|
||||
|
||||
/// <summary>Server version string.</summary>
|
||||
public const string Version = "2.12.0";
|
||||
|
||||
/// <summary>Metadata key for the minimum required API level to use this asset.</summary>
|
||||
public const string RequiredLevelKey = "_nats.req.level";
|
||||
|
||||
/// <summary>Metadata key for the server version that last modified this asset.</summary>
|
||||
public const string ServerVersionKey = "_nats.ver";
|
||||
|
||||
/// <summary>Metadata key for the server API level that last modified this asset.</summary>
|
||||
public const string ServerLevelKey = "_nats.level";
|
||||
|
||||
/// <summary>
|
||||
/// Returns the required API level string from metadata, or empty if absent.
|
||||
/// Go: getRequiredApiLevel (jetstream_versioning.go:28)
|
||||
/// </summary>
|
||||
public static string GetRequiredApiLevel(Dictionary<string, string>? metadata)
|
||||
{
|
||||
if (metadata != null && metadata.TryGetValue(RequiredLevelKey, out var level) && level.Length > 0)
|
||||
return level;
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns whether the required API level is supported by this server.
|
||||
/// Go: supportsRequiredApiLevel (jetstream_versioning.go:36)
|
||||
/// </summary>
|
||||
public static bool SupportsRequiredApiLevel(Dictionary<string, string>? metadata)
|
||||
{
|
||||
var level = GetRequiredApiLevel(metadata);
|
||||
if (level.Length == 0)
|
||||
return true;
|
||||
if (!int.TryParse(level, out var li))
|
||||
return false;
|
||||
return li <= JsApiLevel;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets static (stored) versioning metadata on a stream config.
|
||||
/// Clears dynamic fields (server version/level) and sets the required API level.
|
||||
/// Go: setStaticStreamMetadata (jetstream_versioning.go:44)
|
||||
/// </summary>
|
||||
public static void SetStaticStreamMetadata(StreamConfig cfg)
|
||||
{
|
||||
if (cfg.Metadata == null)
|
||||
cfg.Metadata = [];
|
||||
else
|
||||
DeleteDynamicMetadata(cfg.Metadata);
|
||||
|
||||
var requiredApiLevel = 0;
|
||||
void Requires(int level) { if (level > requiredApiLevel) requiredApiLevel = level; }
|
||||
|
||||
// TTLs require API level 1 (added v2.11)
|
||||
if (cfg.AllowMsgTtl || cfg.SubjectDeleteMarkerTtlMs > 0)
|
||||
Requires(1);
|
||||
|
||||
// Counter CRDTs require API level 2 (added v2.12)
|
||||
if (cfg.AllowMsgCounter)
|
||||
Requires(2);
|
||||
|
||||
// Atomic batch publishing requires API level 2 (added v2.12)
|
||||
if (cfg.AllowAtomicPublish)
|
||||
Requires(2);
|
||||
|
||||
// Message scheduling requires API level 2 (added v2.12)
|
||||
if (cfg.AllowMsgSchedules)
|
||||
Requires(2);
|
||||
|
||||
// Async persist mode requires API level 2 (added v2.12)
|
||||
if (cfg.PersistMode == PersistMode.Async)
|
||||
Requires(2);
|
||||
|
||||
cfg.Metadata[RequiredLevelKey] = requiredApiLevel.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a copy of the stream config with dynamic metadata fields added.
|
||||
/// The original config is not modified.
|
||||
/// Go: setDynamicStreamMetadata (jetstream_versioning.go:88)
|
||||
/// </summary>
|
||||
public static StreamConfig SetDynamicStreamMetadata(StreamConfig cfg)
|
||||
{
|
||||
// Shallow copy the config
|
||||
var newCfg = ShallowCopyStream(cfg);
|
||||
newCfg.Metadata = [];
|
||||
if (cfg.Metadata != null)
|
||||
{
|
||||
foreach (var (k, v) in cfg.Metadata)
|
||||
newCfg.Metadata[k] = v;
|
||||
}
|
||||
newCfg.Metadata[ServerVersionKey] = Version;
|
||||
newCfg.Metadata[ServerLevelKey] = JsApiLevel.ToString();
|
||||
return newCfg;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies versioning fields from prevCfg into cfg (for stream update equality checks).
|
||||
/// Removes dynamic fields. If prevCfg has no metadata, removes the key from cfg.
|
||||
/// Go: copyStreamMetadata (jetstream_versioning.go:110)
|
||||
/// </summary>
|
||||
public static void CopyStreamMetadata(StreamConfig cfg, StreamConfig? prevCfg)
|
||||
{
|
||||
if (cfg.Metadata != null)
|
||||
DeleteDynamicMetadata(cfg.Metadata);
|
||||
SetOrDeleteInStreamMetadata(cfg, prevCfg, RequiredLevelKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets static (stored) versioning metadata on a consumer config.
|
||||
/// Go: setStaticConsumerMetadata (jetstream_versioning.go:136)
|
||||
/// </summary>
|
||||
public static void SetStaticConsumerMetadata(ConsumerConfig cfg)
|
||||
{
|
||||
if (cfg.Metadata == null)
|
||||
cfg.Metadata = [];
|
||||
else
|
||||
DeleteDynamicMetadata(cfg.Metadata);
|
||||
|
||||
var requiredApiLevel = 0;
|
||||
void Requires(int level) { if (level > requiredApiLevel) requiredApiLevel = level; }
|
||||
|
||||
// PauseUntil (non-zero) requires API level 1 (added v2.11)
|
||||
if (cfg.PauseUntil.HasValue && cfg.PauseUntil.Value != DateTime.MinValue && cfg.PauseUntil.Value != default)
|
||||
Requires(1);
|
||||
|
||||
// Priority policy / groups / pinned TTL require API level 1
|
||||
if (cfg.PriorityPolicy != PriorityPolicy.None || cfg.PinnedTtlMs != 0 || cfg.PriorityGroups.Count > 0)
|
||||
Requires(1);
|
||||
|
||||
cfg.Metadata[RequiredLevelKey] = requiredApiLevel.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a copy of the consumer config with dynamic metadata fields added.
|
||||
/// Go: setDynamicConsumerMetadata (jetstream_versioning.go:164)
|
||||
/// </summary>
|
||||
public static ConsumerConfig SetDynamicConsumerMetadata(ConsumerConfig cfg)
|
||||
{
|
||||
var newCfg = ShallowCopyConsumer(cfg);
|
||||
newCfg.Metadata = [];
|
||||
if (cfg.Metadata != null)
|
||||
{
|
||||
foreach (var (k, v) in cfg.Metadata)
|
||||
newCfg.Metadata[k] = v;
|
||||
}
|
||||
newCfg.Metadata[ServerVersionKey] = Version;
|
||||
newCfg.Metadata[ServerLevelKey] = JsApiLevel.ToString();
|
||||
return newCfg;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies versioning fields from prevCfg into cfg (for consumer update equality checks).
|
||||
/// Removes dynamic fields.
|
||||
/// Go: copyConsumerMetadata (jetstream_versioning.go:198)
|
||||
/// </summary>
|
||||
public static void CopyConsumerMetadata(ConsumerConfig cfg, ConsumerConfig? prevCfg)
|
||||
{
|
||||
if (cfg.Metadata != null)
|
||||
DeleteDynamicMetadata(cfg.Metadata);
|
||||
SetOrDeleteInConsumerMetadata(cfg, prevCfg, RequiredLevelKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes dynamic metadata fields (server version and level) from a metadata dictionary.
|
||||
/// Go: deleteDynamicMetadata (jetstream_versioning.go:222)
|
||||
/// </summary>
|
||||
public static void DeleteDynamicMetadata(Dictionary<string, string> metadata)
|
||||
{
|
||||
metadata.Remove(ServerVersionKey);
|
||||
metadata.Remove(ServerLevelKey);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Private helpers
|
||||
// =========================================================================
|
||||
|
||||
private static void SetOrDeleteInStreamMetadata(StreamConfig cfg, StreamConfig? prevCfg, string key)
|
||||
{
|
||||
if (prevCfg?.Metadata != null && prevCfg.Metadata.TryGetValue(key, out var value))
|
||||
{
|
||||
cfg.Metadata ??= [];
|
||||
cfg.Metadata[key] = value;
|
||||
return;
|
||||
}
|
||||
if (cfg.Metadata != null)
|
||||
{
|
||||
cfg.Metadata.Remove(key);
|
||||
if (cfg.Metadata.Count == 0)
|
||||
cfg.Metadata = null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void SetOrDeleteInConsumerMetadata(ConsumerConfig cfg, ConsumerConfig? prevCfg, string key)
|
||||
{
|
||||
if (prevCfg?.Metadata != null && prevCfg.Metadata.TryGetValue(key, out var value))
|
||||
{
|
||||
cfg.Metadata ??= [];
|
||||
cfg.Metadata[key] = value;
|
||||
return;
|
||||
}
|
||||
if (cfg.Metadata != null)
|
||||
{
|
||||
cfg.Metadata.Remove(key);
|
||||
if (cfg.Metadata.Count == 0)
|
||||
cfg.Metadata = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Shallow copy of a StreamConfig for metadata mutation.</summary>
|
||||
private static StreamConfig ShallowCopyStream(StreamConfig src) => new()
|
||||
{
|
||||
Name = src.Name,
|
||||
Description = src.Description,
|
||||
Subjects = src.Subjects,
|
||||
MaxMsgs = src.MaxMsgs,
|
||||
MaxBytes = src.MaxBytes,
|
||||
MaxMsgsPer = src.MaxMsgsPer,
|
||||
MaxAgeMs = src.MaxAgeMs,
|
||||
MaxMsgSize = src.MaxMsgSize,
|
||||
MaxConsumers = src.MaxConsumers,
|
||||
DuplicateWindowMs = src.DuplicateWindowMs,
|
||||
Sealed = src.Sealed,
|
||||
DenyDelete = src.DenyDelete,
|
||||
DenyPurge = src.DenyPurge,
|
||||
AllowDirect = src.AllowDirect,
|
||||
AllowMsgTtl = src.AllowMsgTtl,
|
||||
FirstSeq = src.FirstSeq,
|
||||
Retention = src.Retention,
|
||||
Discard = src.Discard,
|
||||
Storage = src.Storage,
|
||||
Replicas = src.Replicas,
|
||||
Mirror = src.Mirror,
|
||||
Source = src.Source,
|
||||
Sources = src.Sources,
|
||||
SubjectTransformSource = src.SubjectTransformSource,
|
||||
SubjectTransformDest = src.SubjectTransformDest,
|
||||
RePublishSource = src.RePublishSource,
|
||||
RePublishDest = src.RePublishDest,
|
||||
RePublishHeadersOnly = src.RePublishHeadersOnly,
|
||||
SubjectDeleteMarkerTtlMs = src.SubjectDeleteMarkerTtlMs,
|
||||
AllowMsgSchedules = src.AllowMsgSchedules,
|
||||
AllowMsgCounter = src.AllowMsgCounter,
|
||||
AllowAtomicPublish = src.AllowAtomicPublish,
|
||||
PersistMode = src.PersistMode,
|
||||
Metadata = src.Metadata,
|
||||
};
|
||||
|
||||
/// <summary>Shallow copy of a ConsumerConfig for metadata mutation.</summary>
|
||||
private static ConsumerConfig ShallowCopyConsumer(ConsumerConfig src) => new()
|
||||
{
|
||||
DurableName = src.DurableName,
|
||||
Ephemeral = src.Ephemeral,
|
||||
FilterSubject = src.FilterSubject,
|
||||
FilterSubjects = src.FilterSubjects,
|
||||
AckPolicy = src.AckPolicy,
|
||||
DeliverPolicy = src.DeliverPolicy,
|
||||
OptStartSeq = src.OptStartSeq,
|
||||
OptStartTimeUtc = src.OptStartTimeUtc,
|
||||
ReplayPolicy = src.ReplayPolicy,
|
||||
AckWaitMs = src.AckWaitMs,
|
||||
MaxDeliver = src.MaxDeliver,
|
||||
MaxAckPending = src.MaxAckPending,
|
||||
Push = src.Push,
|
||||
DeliverSubject = src.DeliverSubject,
|
||||
HeartbeatMs = src.HeartbeatMs,
|
||||
BackOffMs = src.BackOffMs,
|
||||
FlowControl = src.FlowControl,
|
||||
RateLimitBps = src.RateLimitBps,
|
||||
MaxWaiting = src.MaxWaiting,
|
||||
MaxRequestBatch = src.MaxRequestBatch,
|
||||
MaxRequestMaxBytes = src.MaxRequestMaxBytes,
|
||||
MaxRequestExpiresMs = src.MaxRequestExpiresMs,
|
||||
PauseUntil = src.PauseUntil,
|
||||
PriorityPolicy = src.PriorityPolicy,
|
||||
PriorityGroups = src.PriorityGroups,
|
||||
PinnedTtlMs = src.PinnedTtlMs,
|
||||
Metadata = src.Metadata,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user