feat(batch37): merge stream-messages

This commit is contained in:
Joseph Doherty
2026-03-01 00:22:33 -05:00
17 changed files with 2066 additions and 10 deletions

View File

@@ -0,0 +1,33 @@
using Shouldly;
using ZB.MOM.NatsNet.Server;
namespace ZB.MOM.NatsNet.Server.Tests.Accounts;
public sealed class AccountStreamRestoreTests
{
[Fact]
public void RestoreStream_EmptySnapshot_ReturnsError()
{
var account = new Account { Name = "A" };
var config = new StreamConfig { Name = "S", Storage = StorageType.MemoryStorage };
var (stream, error) = account.RestoreStream(config, new MemoryStream());
stream.ShouldBeNull();
error.ShouldNotBeNull();
}
[Fact]
public void RestoreStream_WithSnapshotData_AddsStream()
{
var account = new Account { Name = "A" };
var config = new StreamConfig { Name = "S", Storage = StorageType.MemoryStorage };
using var snapshot = new MemoryStream([1, 2, 3]);
var (stream, error) = account.RestoreStream(config, snapshot);
error.ShouldBeNull();
stream.ShouldNotBeNull();
stream!.Name.ShouldBe("S");
}
}

View File

@@ -0,0 +1,181 @@
using NSubstitute;
using Shouldly;
using ZB.MOM.NatsNet.Server;
using ZB.MOM.NatsNet.Server.Internal;
namespace ZB.MOM.NatsNet.Server.Tests.JetStream;
public sealed class Batch37StreamMessagesMappedTests
{
[Fact] // T:828
public void JetStreamClusterStreamDirectGetMsg_ShouldSucceed()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
stream.Store!.StoreMsg("orders.created", null, [1], 0);
var request = System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(new JsApiMsgGetRequest { Seq = 1 });
var response = stream.ProcessDirectGetRequest("reply", null, request);
response.Error.ShouldBeNull();
response.Message.ShouldNotBeNull();
response.Message!.Sequence.ShouldBe(1UL);
}
[Fact] // T:954
public void JetStreamClusterRollupSubjectAndWatchers_ShouldSucceed()
{
var hdr = NatsMessageHeaders.GenHeader(null, NatsHeaderConstants.JsMsgRollup, "SUB");
NatsStream.GetRollup(hdr).ShouldBe("sub");
}
[Fact] // T:987
public void JetStreamClusterMirrorDeDupWindow_ShouldSucceed()
{
var stream = CreateStream(duplicates: TimeSpan.FromSeconds(5));
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() * 1_000_000L;
stream.StoreMsgId(new NatsStream.DedupeEntry { Id = "dup-1", Seq = 10, TimestampNanos = now });
stream.CheckMsgId("dup-1").ShouldNotBeNull();
stream.NumMsgIds().ShouldBe(1);
}
[Fact] // T:1642
public void JetStreamDirectMsgGet_ShouldSucceed()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
stream.ProcessJetStreamMsg("events", string.Empty, null, [7], 0, 0, null, false, true).ShouldBeNull();
var req = System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(new JsApiMsgGetRequest { Seq = 1 });
var response = stream.ProcessDirectGetRequest("reply", null, req);
response.Message.ShouldNotBeNull();
response.Message!.Data.ShouldBe([7]);
}
[Fact] // T:1643
public void JetStreamDirectMsgGetNext_ShouldSucceed()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
stream.ProcessJetStreamMsg("events.a", string.Empty, null, [1], 0, 0, null, false, true).ShouldBeNull();
stream.ProcessJetStreamMsg("events.a", string.Empty, null, [2], 0, 0, null, false, true).ShouldBeNull();
var response = stream.GetDirectRequest(new JsApiMsgGetRequest { NextFor = "events.*", Seq = 1 }, "reply");
response.Message.ShouldNotBeNull();
response.Message!.Sequence.ShouldBeGreaterThanOrEqualTo(1UL);
}
[Fact] // T:383
public void FileStoreSnapshot_ShouldSucceed()
{
var root = Path.Combine(Path.GetTempPath(), $"batch37-snap-{Guid.NewGuid():N}");
Directory.CreateDirectory(root);
try
{
var stream = CreateStream(storage: StorageType.FileStorage, storeDir: root);
stream.SetupStore(new FileStoreConfig { StoreDir = root }).ShouldBeNull();
stream.ProcessJetStreamMsg("snap.a", string.Empty, null, [1], 0, 0, null, false, true).ShouldBeNull();
var (result, error) = stream.Snapshot(TimeSpan.FromSeconds(2), checkMsgs: false, includeConsumers: false);
error.ShouldBeNull();
result.ShouldNotBeNull();
}
finally
{
if (Directory.Exists(root))
Directory.Delete(root, recursive: true);
}
}
[Fact] // T:2617
public void NRGSnapshotAndRestart_ShouldSucceed()
{
var raft = new Raft { GroupName = "RG" };
raft.InstallSnapshot([1, 2, 3], force: true);
raft.Wps.ShouldBe([1, 2, 3]);
}
[Fact] // T:2672
public void NRGSnapshotCatchup_ShouldSucceed()
{
var raft = new Raft { GroupName = "RG", StateValue = (int)RaftState.Leader };
raft.SendSnapshot([4, 5, 6]);
raft.Wps.ShouldBe([4, 5, 6]);
}
[Fact] // T:2695
public void NRGNoLogResetOnCorruptedSendToFollower_ShouldSucceed()
{
var raft = new Raft { GroupName = "RG", StateValue = (int)RaftState.Leader };
Should.NotThrow(() => raft.SendSnapshot([0, 0, 0]));
}
[Fact] // T:635
public void GatewayQueueSub_ShouldSucceed()
{
var outq = new JsOutQ();
outq.SendMsg("inbox.gateway", [1, 2]).Error.ShouldBeNull();
outq.Pop().ShouldNotBeNull();
}
[Fact] // T:1892
public void JWTImportsOnServerRestartAndClientsReconnect_ShouldSucceed()
{
var account = new Account { Name = "A" };
var cfg = new StreamConfig { Name = "RESTORE", Storage = StorageType.MemoryStorage };
using var snapshot = new MemoryStream([1, 2, 3, 4]);
var (stream, error) = account.RestoreStream(cfg, snapshot);
error.ShouldBeNull();
stream.ShouldNotBeNull();
}
[Fact] // T:1961
public void LeafNodeQueueGroupDistributionWithDaisyChainAndGateway_ShouldSucceed()
{
var account = Account.NewAccount("L");
var leaf1 = new ClientConnection(ClientKind.Leaf);
var leaf2 = new ClientConnection(ClientKind.Leaf);
((INatsAccount)account).AddClient(leaf1);
((INatsAccount)account).AddClient(leaf2);
account.NumLocalLeafNodes().ShouldBe(2);
}
[Fact] // T:2426
public void NoRaceJetStreamConsumerFilterPerfDegradation_ShouldSucceed()
{
var stream = CreateStream(subjects: ["perf.>"]);
stream.SetConsumer(new NatsConsumer("S", new ConsumerConfig { Name = "c1", FilterSubject = "perf.*" }, DateTime.UtcNow));
stream.PotentialFilteredConsumers().ShouldBeTrue();
stream.PartitionUnique("c2", ["perf.x"]).ShouldBeFalse();
}
private static NatsStream CreateStream(
TimeSpan? duplicates = null,
StorageType storage = StorageType.MemoryStorage,
string[]? subjects = null,
string? storeDir = null)
{
var account = new Account { Name = "A" };
if (!string.IsNullOrWhiteSpace(storeDir))
account.JetStream = new JsAccount { StoreDir = storeDir };
var config = new StreamConfig
{
Name = "S",
Storage = storage,
Subjects = subjects ?? ["events.>"],
Retention = RetentionPolicy.InterestPolicy,
Duplicates = duplicates ?? TimeSpan.FromSeconds(1),
};
return new NatsStream(account, config, DateTime.UtcNow);
}
}

View File

@@ -0,0 +1,142 @@
using Shouldly;
using ZB.MOM.NatsNet.Server;
namespace ZB.MOM.NatsNet.Server.Tests.JetStream;
public sealed class NatsStreamConsumersTests
{
[Fact]
public void NewCMsg_ReturnToPool_ClearsValues()
{
var msg = NatsStream.NewCMsg("orders.created", 22);
msg.Subject.ShouldBe("orders.created");
msg.Seq.ShouldBe(22UL);
msg.ReturnToPool();
msg.Subject.ShouldBe(string.Empty);
msg.Seq.ShouldBe(0UL);
}
[Fact]
public void NewJSPubMsg_WithHeaderAndData_ReturnsSizedMessage()
{
var pub = NatsStream.NewJSPubMsg("inbox.x", "orders", "reply", [1, 2], [3, 4, 5], null, 12);
pub.Subject.ShouldBe("inbox.x");
pub.Size().ShouldBeGreaterThan(0);
}
[Fact]
public void JsOutQ_SendThenUnregister_RejectsFutureSends()
{
var outq = new JsOutQ();
var first = outq.SendMsg("inbox.1", [1, 2, 3]);
first.Error.ShouldBeNull();
first.Len.ShouldBeGreaterThan(0);
outq.Unregister();
var second = outq.SendMsg("inbox.2", [4]);
second.Error.ShouldNotBeNull();
}
[Fact]
public void AccName_AndNameLocked_ReturnConfiguredValues()
{
var stream = CreateStream();
stream.AccName().ShouldBe("A");
stream.NameLocked().ShouldBe("S");
}
[Fact]
public void SetupSendCapabilities_AndResetConsumers_DoNotThrow()
{
var stream = CreateStream();
Should.NotThrow(stream.SetupSendCapabilities);
Should.NotThrow(stream.ResetAndWaitOnConsumers);
}
[Fact]
public void SetConsumer_LookupAndCounts_ReturnExpectedValues()
{
var stream = CreateStream();
var standard = new NatsConsumer("S", new ConsumerConfig { Name = "c1", Direct = false }, DateTime.UtcNow);
var direct = new NatsConsumer("S", new ConsumerConfig { Name = "c2", Direct = true }, DateTime.UtcNow);
stream.SetConsumer(standard);
stream.SetConsumer(direct);
stream.NumConsumers().ShouldBe(2);
stream.NumPublicConsumers().ShouldBe(1);
stream.NumDirectConsumers().ShouldBe(1);
stream.LookupConsumer("c2").ShouldBe(direct);
}
[Fact]
public void GetMsg_WhenStored_ReturnsStoredMessage()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
stream.Store!.StoreMsg("events", null, [1, 2], ttl: 0);
var message = stream.GetMsg(1);
message.ShouldNotBeNull();
message!.Subject.ShouldBe("events");
message.Sequence.ShouldBe(1UL);
}
[Fact]
public void PartitionUnique_WithCollidingFilters_ReturnsFalse()
{
var stream = CreateStream();
var existing = new NatsConsumer("S", new ConsumerConfig { Name = "existing", FilterSubject = "orders.*" }, DateTime.UtcNow);
stream.SetConsumer(existing);
var unique = stream.PartitionUnique("new", ["orders.created"]);
unique.ShouldBeFalse();
}
[Fact]
public void PotentialFilteredConsumers_WithWildcardSubjectAndConsumer_ReturnsTrue()
{
var stream = CreateStream(["orders.>"]);
stream.SetConsumer(new NatsConsumer("S", new ConsumerConfig { Name = "c1" }, DateTime.UtcNow));
stream.PotentialFilteredConsumers().ShouldBeTrue();
}
[Fact]
public void NoInterest_WithOnlyObservingConsumer_ReturnsTrue()
{
var stream = CreateStream();
var observer = new NatsConsumer("S", new ConsumerConfig { Name = "observer" }, DateTime.UtcNow);
stream.SetConsumer(observer);
stream.NoInterest(1, observer).ShouldBeTrue();
}
[Fact]
public void IsInterestRetention_WhenPolicyIsInterest_ReturnsTrue()
{
var stream = CreateStream(retention: RetentionPolicy.InterestPolicy);
stream.IsInterestRetention().ShouldBeTrue();
}
private static NatsStream CreateStream(string[]? subjects = null, RetentionPolicy retention = RetentionPolicy.LimitsPolicy)
{
var account = new Account { Name = "A" };
var config = new StreamConfig
{
Name = "S",
Storage = StorageType.MemoryStorage,
Subjects = subjects ?? ["events.>"],
Retention = retention,
};
return new NatsStream(account, config, DateTime.UtcNow);
}
}

View File

@@ -0,0 +1,152 @@
using System.Text;
using System.Text.Json;
using NSubstitute;
using Shouldly;
using ZB.MOM.NatsNet.Server;
using ZB.MOM.NatsNet.Server.Internal;
namespace ZB.MOM.NatsNet.Server.Tests.JetStream;
public sealed class NatsStreamDirectGetPipelineTests
{
[Fact]
public void GetMessageScheduleTTL_InvalidValue_ReturnsNotOk()
{
var hdr = NatsMessageHeaders.GenHeader(null, NatsHeaderConstants.JsScheduleTtl, "not-a-ttl");
var (ttl, ok) = NatsStream.GetMessageScheduleTTL(hdr);
ok.ShouldBeFalse();
ttl.ShouldBe(string.Empty);
}
[Fact]
public void GetBatchSequence_ValidHeader_ReturnsSequence()
{
var hdr = NatsMessageHeaders.GenHeader(null, NatsHeaderConstants.JsBatchSeq, "9");
var (seq, ok) = NatsStream.GetBatchSequence(hdr);
ok.ShouldBeTrue();
seq.ShouldBe(9UL);
}
[Fact]
public void IsClustered_WithAssignmentNode_ReturnsTrue()
{
var stream = CreateStream();
var node = Substitute.For<IRaftNode>();
var assignment = new StreamAssignment
{
Group = new RaftGroup
{
Node = node,
Peers = ["N1"],
},
};
stream.SetStreamAssignment(assignment);
stream.IsClustered().ShouldBeTrue();
stream.IsClusteredInternal().ShouldBeTrue();
}
[Fact]
public void InMsgReturnToPool_WithValues_ClearsState()
{
var msg = new InMsg
{
Subject = "foo",
Reply = "bar",
Hdr = [1, 2],
Msg = [3, 4],
Client = new object(),
};
msg.ReturnToPool();
msg.Subject.ShouldBe(string.Empty);
msg.Reply.ShouldBeNull();
msg.Hdr.ShouldBeNull();
msg.Msg.ShouldBeNull();
}
[Fact]
public void QueueInbound_WithQueue_PushesMessage()
{
var stream = CreateStream();
var queue = new IpQueue<InMsg>("inbound");
stream.QueueInbound(queue, "orders.created", null, null, [1], null, null);
var popped = queue.Pop();
popped.ShouldNotBeNull();
popped!.Length.ShouldBe(1);
popped[0].Subject.ShouldBe("orders.created");
}
[Fact]
public void ProcessDirectGetRequest_SeqRequest_ReturnsStoredMessage()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
stream.Store!.StoreMsg("orders", null, [7, 8], ttl: 0);
var request = JsonSerializer.SerializeToUtf8Bytes(new JsApiMsgGetRequest { Seq = 1 });
var response = stream.ProcessDirectGetRequest("reply.inbox", null, request);
response.Error.ShouldBeNull();
response.Message.ShouldNotBeNull();
response.Message!.Sequence.ShouldBe(1UL);
response.Message.Subject.ShouldBe("orders");
}
[Fact]
public void ProcessDirectGetLastBySubjectRequest_InvalidSubject_ReturnsBadRequest()
{
var stream = CreateStream();
var response = stream.ProcessDirectGetLastBySubjectRequest("$JS.API.DIRECT.GET.STREAM", "reply", null, null);
response.Error.ShouldNotBeNull();
response.Error!.ErrCode.ShouldBe(JsApiErrors.BadRequest.ErrCode);
}
[Fact]
public void ProcessJetStreamMsg_WithMemoryStore_StoresMessage()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
var error = stream.ProcessJetStreamMsg("events", string.Empty, null, Encoding.ASCII.GetBytes("m1"), 0, 0, null, false, true);
var stored = stream.Store!.LoadMsg(1, new StoreMsg());
error.ShouldBeNull();
stored.ShouldNotBeNull();
stored!.Subject.ShouldBe("events");
}
[Fact]
public void ProcessJetStreamBatchMsg_MissingSequence_ReturnsApiError()
{
var stream = CreateStream();
stream.SetupStore(null).ShouldBeNull();
var error = stream.ProcessJetStreamBatchMsg("batch-1", "events", string.Empty, null, [1], null);
error.ShouldNotBeNull();
error.ShouldBeOfType<InvalidOperationException>();
}
private static NatsStream CreateStream()
{
var account = new Account { Name = "A" };
var config = new StreamConfig
{
Name = "S",
Storage = StorageType.MemoryStorage,
Subjects = ["events.>"],
};
return new NatsStream(account, config, DateTime.UtcNow);
}
}

View File

@@ -0,0 +1,79 @@
using System.Numerics;
using Shouldly;
using ZB.MOM.NatsNet.Server;
namespace ZB.MOM.NatsNet.Server.Tests.JetStream;
public sealed class NatsStreamMessageHeadersTests
{
[Fact]
public void ParseMessageTTL_NeverValue_ReturnsMinusOne()
{
var (ttl, error) = NatsStream.ParseMessageTTL("never");
error.ShouldBeNull();
ttl.ShouldBe(-1);
}
[Fact]
public void GetMessageIncr_ValidHeader_ReturnsParsedValue()
{
var hdr = NatsMessageHeaders.GenHeader(null, NatsHeaderConstants.JsMessageIncr, "42");
var (value, ok) = NatsStream.GetMessageIncr(hdr);
ok.ShouldBeTrue();
value.ShouldBe(new BigInteger(42));
}
[Fact]
public void GetMessageSchedule_InvalidPattern_ReturnsNotOk()
{
var hdr = NatsMessageHeaders.GenHeader(null, NatsHeaderConstants.JsSchedulePattern, "invalid");
var (schedule, ok) = NatsStream.GetMessageSchedule(hdr);
ok.ShouldBeFalse();
schedule.ShouldBe(default);
}
[Fact]
public void SetupStore_MemoryStorage_CreatesMemStore()
{
var stream = CreateStream(duplicates: TimeSpan.FromSeconds(1));
var error = stream.SetupStore(null);
error.ShouldBeNull();
stream.Store.ShouldNotBeNull();
stream.Store!.Type().ShouldBe(StorageType.MemoryStorage);
}
[Fact]
public void StoreMsgIdAndPurge_ExpiredEntry_RemovesEntry()
{
var stream = CreateStream(duplicates: TimeSpan.FromMilliseconds(50));
var oldTs = (DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1)).ToUnixTimeMilliseconds() * 1_000_000L;
stream.StoreMsgId(new NatsStream.DedupeEntry { Id = "id-1", Seq = 1, TimestampNanos = oldTs });
stream.NumMsgIds().ShouldBe(1);
stream.CheckMsgId("id-1").ShouldNotBeNull();
stream.PurgeMsgIds();
stream.NumMsgIds().ShouldBe(0);
stream.CheckMsgId("id-1").ShouldBeNull();
}
private static NatsStream CreateStream(TimeSpan duplicates)
{
var account = new Account { Name = "A" };
var config = new StreamConfig
{
Name = "S",
Storage = StorageType.MemoryStorage,
Duplicates = duplicates,
};
return new NatsStream(account, config, DateTime.UtcNow);
}
}

View File

@@ -0,0 +1,102 @@
using NSubstitute;
using Shouldly;
using ZB.MOM.NatsNet.Server;
namespace ZB.MOM.NatsNet.Server.Tests.JetStream;
public sealed class NatsStreamSnapshotMonitorTests
{
[Fact]
public void RegisterPreAck_ClearPreAck_UpdatesState()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
var consumer = new NatsConsumer("S", new ConsumerConfig { Name = "c1" }, DateTime.UtcNow);
stream.RegisterPreAck(consumer, 2);
stream.HasPreAck(consumer, 2).ShouldBeTrue();
stream.ClearPreAck(consumer, 2);
stream.HasPreAck(consumer, 2).ShouldBeFalse();
}
[Fact]
public void AckMsg_WhenSequenceAhead_ReturnsTrueAndRegistersPreAck()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
stream.SetupStore(null).ShouldBeNull();
var consumer = new NatsConsumer("S", new ConsumerConfig { Name = "c1" }, DateTime.UtcNow);
var removed = stream.AckMsg(consumer, 50);
removed.ShouldBeTrue();
stream.HasPreAck(consumer, 50).ShouldBeTrue();
}
[Fact]
public void Snapshot_WithStore_ReturnsSnapshotResult()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
stream.SetupStore(null).ShouldBeNull();
stream.Store!.StoreMsg("events", null, [1], ttl: 0);
var (result, error) = stream.Snapshot(TimeSpan.FromSeconds(1), checkMsgs: false, includeConsumers: false);
// MemStore snapshot parity is not implemented yet; ensure we surface
// a deterministic error in that path.
if (stream.Store.Type() == StorageType.MemoryStorage)
{
error.ShouldNotBeNull();
result.ShouldBeNull();
}
else
{
error.ShouldBeNull();
result.ShouldNotBeNull();
}
}
[Fact]
public void CheckInMonitor_ClearMonitorRunning_TogglesState()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
stream.CheckInMonitor().ShouldBeFalse();
stream.IsMonitorRunning().ShouldBeTrue();
stream.ClearMonitorRunning();
stream.IsMonitorRunning().ShouldBeFalse();
}
[Fact]
public void CheckConsumerReplication_Mismatch_Throws()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
stream.Config.Replicas = 3;
stream.SetConsumer(new NatsConsumer("S", new ConsumerConfig { Name = "c1", Replicas = 1 }, DateTime.UtcNow));
Should.Throw<InvalidOperationException>(stream.CheckConsumerReplication);
}
[Fact]
public void TrackReplicationTraffic_SystemAccountNode_DoesNotThrow()
{
var stream = CreateStream(RetentionPolicy.InterestPolicy);
var node = Substitute.For<IRaftNode>();
node.IsSystemAccount().Returns(true);
Should.NotThrow(() => stream.TrackReplicationTraffic(node, size: 256, replicas: 3));
}
private static NatsStream CreateStream(RetentionPolicy retention)
{
var account = new Account { Name = "A" };
var config = new StreamConfig
{
Name = "S",
Storage = StorageType.MemoryStorage,
Subjects = ["events.>"],
Retention = retention,
};
return new NatsStream(account, config, DateTime.UtcNow);
}
}