dohertj2/natsnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(batch15): complete group 4 msgblock/consumerfilestore

Browse Source
This commit is contained in:
Joseph Doherty
2026-02-28 18:03:56 -05:00
parent ced3e7e9b8
commit 43d914bf83
6 changed files with 939 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
dotnet/tests/ZB.MOM.NatsNet.Server.Tests/ImplBacklog/ConcurrencyTests1.Impltests.cs
View File

@@ -7,6 +7,46 @@ namespace ZB.MOM.NatsNet.Server.Tests.ImplBacklog;
public sealed partial class ConcurrencyTests1
{
[Fact] // T:2422
public void NoRaceJetStreamConsumerFileStoreConcurrentDiskIO_ShouldSucceed()
{
WithStore((fs, _) =>
{
const int consumerCount = 400;
var start = new ManualResetEventSlim(false);
var errors = new ConcurrentQueue<Exception>();
var timestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L;
var workers = new List<Task>(consumerCount);
for (var i = 0; i < consumerCount; i++)
{
var consumer = fs.ConsumerStore(
$"o{i}",
DateTime.UtcNow,
new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit });
workers.Add(Task.Run(() =>
{
try
{
start.Wait(TimeSpan.FromSeconds(5));
consumer.UpdateDelivered(22, 22, 1, timestamp);
consumer.EncodedState();
consumer.Delete();
}
catch (Exception ex)
{
errors.Enqueue(ex);
}
}));
}
start.Set();
Task.WaitAll(workers.ToArray());
errors.ShouldBeEmpty();
});
}
[Fact] // T:2452
public void NoRaceFileStoreStreamMaxAgePerformance_ShouldSucceed()
{

158
dotnet/tests/ZB.MOM.NatsNet.Server.Tests/ImplBacklog/ConcurrencyTests2.Impltests.cs
View File

@@ -8,6 +8,145 @@ namespace ZB.MOM.NatsNet.Server.Tests.ImplBacklog;
public sealed partial class ConcurrencyTests2
{
[Fact] // T:2505
public void NoRaceStoreReverseWalkWithDeletesPerf_ShouldSucceed()
{
var root = NewRoot();
Directory.CreateDirectory(root);
var fileCfg = new StreamConfig
{
Name = "zzz",
Subjects = ["foo.*"],
Storage = StorageType.FileStorage,
MaxMsgs = -1,
MaxBytes = -1,
MaxAge = TimeSpan.Zero,
MaxMsgsPer = -1,
Discard = DiscardPolicy.DiscardOld,
Retention = RetentionPolicy.LimitsPolicy,
};
var memCfg = fileCfg.Clone();
memCfg.Storage = StorageType.MemoryStorage;
var fs = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, fileCfg);
var ms = JetStreamMemStore.NewMemStore(memCfg);
try
{
var msg = "Hello"u8.ToArray();
foreach (var store in new IStreamStore[] { fs, ms })
{
store.StoreMsg("foo.A", null, msg, 0).Seq.ShouldBeGreaterThan(0UL);
for (var i = 0; i < 150_000; i++)
store.StoreMsg("foo.B", null, msg, 0);
store.StoreMsg("foo.C", null, msg, 0);
var state = store.State();
state.Msgs.ShouldBe(150_002UL);
var (purged, purgeErr) = store.PurgeEx("foo.B", 1, 0);
purgeErr.ShouldBeNull();
purged.ShouldBe(150_000UL);
if (store is JetStreamFileStore fileStore)
PreloadFileStoreCaches(fileStore);
var timer = Stopwatch.StartNew();
var scratch = new StoreMsg();
for (var seq = state.LastSeq; seq > 0; seq--)
{
try
{
_ = store.LoadMsg(seq, scratch);
}
catch (Exception ex) when (ReferenceEquals(ex, StoreErrors.ErrStoreMsgNotFound))
{
continue;
}
}
timer.Stop();
var reverseWalkElapsed = timer.Elapsed;
if (store is JetStreamFileStore fileStore2)
PreloadFileStoreCaches(fileStore2);
var seen = 0;
var cursor = state.LastSeq;
timer.Restart();
while (true)
{
var (sm, err) = store.LoadPrevMsg(cursor, scratch);
if (err == StoreErrors.ErrStoreEOF)
break;
err.ShouldBeNull();
sm.ShouldNotBeNull();
cursor = sm!.Seq > 0 ? sm.Seq - 1 : 0;
seen++;
}
timer.Stop();
seen.ShouldBe(2);
if (store is JetStreamMemStore)
{
timer.Elapsed.ShouldBeLessThan(reverseWalkElapsed);
}
else
{
(timer.Elapsed.Ticks * 10L).ShouldBeLessThan(reverseWalkElapsed.Ticks);
}
}
}
finally
{
fs.Stop();
ms.Stop();
if (Directory.Exists(root))
Directory.Delete(root, recursive: true);
}
}
[Fact] // T:2510
public void NoRaceFileStorePurgeExAsyncTombstones_ShouldSucceed()
{
var cfg = DefaultStreamConfig();
cfg.Subjects = ["*.*"];
WithStore((fs, _) =>
{
var msg = "zzz"u8.ToArray();
fs.StoreMsg("foo.A", null, msg, 0);
fs.StoreMsg("foo.B", null, msg, 0);
for (var i = 0; i < 500; i++)
fs.StoreMsg("foo.C", null, msg, 0);
fs.StoreMsg("foo.D", null, msg, 0);
PreloadFileStoreCaches(fs);
var sw = Stopwatch.StartNew();
var (purgedOne, errOne) = fs.PurgeEx("foo.B", 0, 0);
sw.Stop();
errOne.ShouldBeNull();
purgedOne.ShouldBe(1UL);
var singleElapsed = sw.Elapsed;
sw.Restart();
var (purgedMany, errMany) = fs.PurgeEx("foo.C", 0, 0);
sw.Stop();
errMany.ShouldBeNull();
purgedMany.ShouldBe(500UL);
var manyElapsed = sw.Elapsed;
// Large subject purges should not degenerate to per-message sync behavior.
var scaledSingle = Math.Max(1L, singleElapsed.Ticks) * 80L;
scaledSingle.ShouldBeGreaterThan(manyElapsed.Ticks);
}, cfg);
}
[Fact] // T:2491
public void NoRaceFileStoreMsgLoadNextMsgMultiPerf_ShouldSucceed()
{
@@ -243,5 +382,24 @@ public sealed partial class ConcurrencyTests2
return (T)result;
}
private static void PreloadFileStoreCaches(JetStreamFileStore fs)
{
var blksField = typeof(JetStreamFileStore).GetField("_blks", BindingFlags.Instance | BindingFlags.NonPublic);
blksField.ShouldNotBeNull();
var blocks = blksField!.GetValue(fs) as System.Collections.IEnumerable;
blocks.ShouldNotBeNull();
foreach (var mb in blocks!)
{
var load = mb!.GetType().GetMethod("LoadMsgs", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
load.ShouldNotBeNull();
var result = load!.Invoke(mb, []);
if (result is Exception err)
throw err;
}
}
private static string NewRoot() => Path.Combine(Path.GetTempPath(), $"impl-fs-c2-{Guid.NewGuid():N}");
}