88a82ee860
Add XML doc comments to public properties across EventTypes, Connz, Varz, NatsOptions, StreamConfig, IStreamStore, FileStore, MqttListener, MqttSessionStore, MessageTraceContext, and JetStreamApiResponse. Fix flaky tests by increasing timing margins (ResponseTracker expiry 1ms→50ms, sleep 50ms→200ms) and document known flaky test patterns in tests.md.
1287 lines
39 KiB
C#
1287 lines
39 KiB
C#
// Go parity: golang/nats-server/server/norace_test.go
|
|
// Covers: race condition tests from Go's norace_test.go - concurrent
|
|
// publish/subscribe, concurrent stream create/delete, concurrent consumer
|
|
// create/ack, parallel message delivery ordering, stress tests for
|
|
// SubList thread safety, concurrent connection open/close.
|
|
using System.Collections.Concurrent;
|
|
using System.Text;
|
|
using NATS.Server.JetStream;
|
|
using NATS.Server.JetStream.Api;
|
|
using NATS.Server.JetStream.Cluster;
|
|
using NATS.Server.JetStream.Consumers;
|
|
using NATS.Server.JetStream.Models;
|
|
using NATS.Server.JetStream.Publish;
|
|
using NATS.Server.JetStream.Storage;
|
|
using NATS.Server.Raft;
|
|
using NATS.Server.Subscriptions;
|
|
|
|
namespace NATS.Server.Core.Tests;
|
|
|
|
/// <summary>
|
|
/// NORACE concurrency stress tests ported from Go's norace_test.go.
|
|
/// These tests use concurrent operations via Parallel.ForEachAsync,
|
|
/// Task.WhenAll, and ConcurrentDictionary to create race conditions
|
|
/// and verify thread safety of core data structures.
|
|
/// </summary>
|
|
public class ConcurrencyStressTests
|
|
{
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistBasic server/norace_test.go (SubList concurrency)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_concurrent_insert_and_match_is_thread_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numOps = 500;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, numOps, i =>
|
|
{
|
|
try
|
|
{
|
|
var sub = new Subscription
|
|
{
|
|
Subject = $"test.subject.{i % 50}",
|
|
Sid = $"sid-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
_ = subList.Match($"test.subject.{i % 50}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
subList.Count.ShouldBeGreaterThan(0U);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistMatch server/norace_test.go (SubList match under load)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_concurrent_insert_remove_and_match_does_not_corrupt()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numSubs = 200;
|
|
|
|
// Pre-populate subscriptions
|
|
var subs = new List<Subscription>();
|
|
for (var i = 0; i < numSubs; i++)
|
|
{
|
|
var sub = new Subscription
|
|
{
|
|
Subject = $"concurrent.{i % 20}.data",
|
|
Sid = $"sid-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
subs.Add(sub);
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Concurrently match while removing
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < numSubs; i++)
|
|
_ = subList.Match($"concurrent.{i % 20}.data");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
foreach (var sub in subs.Take(numSubs / 2))
|
|
subList.Remove(sub);
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
var newSub = new Subscription
|
|
{
|
|
Subject = $"concurrent.new.{i}",
|
|
Sid = $"new-{i}",
|
|
};
|
|
subList.Insert(newSub);
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistWildcard server/norace_test.go (wildcard matching)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_concurrent_wildcard_insert_and_match_is_thread_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numOps = 300;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, numOps, (int i) =>
|
|
{
|
|
try
|
|
{
|
|
var subjectPattern = (i % 3) switch
|
|
{
|
|
0 => $"wc.{i % 10}.*",
|
|
1 => $"wc.{i % 10}.>",
|
|
_ => $"wc.{i % 10}.literal",
|
|
};
|
|
var sub = new Subscription
|
|
{
|
|
Subject = subjectPattern,
|
|
Sid = $"wc-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
_ = subList.Match($"wc.{i % 10}.test");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistQueueSub server/norace_test.go (queue subs)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_concurrent_queue_group_operations_are_thread_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numOps = 200;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, numOps, i =>
|
|
{
|
|
try
|
|
{
|
|
var sub = new Subscription
|
|
{
|
|
Subject = $"queue.subject.{i % 10}",
|
|
Queue = $"group-{i % 5}",
|
|
Sid = $"qsid-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
var result = subList.Match($"queue.subject.{i % 10}");
|
|
// Queue subs should be grouped
|
|
if (result.QueueSubs.Length > 0)
|
|
{
|
|
foreach (var group in result.QueueSubs)
|
|
group.Length.ShouldBeGreaterThan(0);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterStreamCreate server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_stream_create_does_not_corrupt_stream_manager()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
const int numStreams = 100;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
var createdStreams = new ConcurrentBag<string>();
|
|
|
|
Parallel.For(0, numStreams, i =>
|
|
{
|
|
try
|
|
{
|
|
var resp = streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = $"STREAM-{i}",
|
|
Subjects = [$"s{i}.>"],
|
|
Replicas = 1,
|
|
});
|
|
if (resp.Error is null)
|
|
createdStreams.Add($"STREAM-{i}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
createdStreams.Count.ShouldBe(numStreams);
|
|
streamManager.StreamNames.Count.ShouldBe(numStreams);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterStreamDelete server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_stream_create_and_delete_is_thread_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
const int numStreams = 50;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// First create streams
|
|
for (var i = 0; i < numStreams; i++)
|
|
{
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = $"CD-{i}",
|
|
Subjects = [$"cd{i}.>"],
|
|
Replicas = 1,
|
|
});
|
|
}
|
|
|
|
// Concurrently delete some and create new ones
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < numStreams / 2; i++)
|
|
streamManager.Delete($"CD-{i}");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = numStreams; i < numStreams + 25; i++)
|
|
{
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = $"CD-{i}",
|
|
Subjects = [$"cd{i}.>"],
|
|
Replicas = 1,
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterConsumerCreate server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_consumer_create_does_not_corrupt()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var consumerManager = new ConsumerManager(meta);
|
|
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "CONC",
|
|
Subjects = ["conc.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
const int numConsumers = 100;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, numConsumers, i =>
|
|
{
|
|
try
|
|
{
|
|
consumerManager.CreateOrUpdate("CONC", new ConsumerConfig
|
|
{
|
|
DurableName = $"consumer-{i}",
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
consumerManager.ConsumerCount.ShouldBe(numConsumers);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterConsumerAck server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_publish_and_consumer_ack_is_thread_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var consumerManager = new ConsumerManager(meta);
|
|
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
|
|
var publisher = new JetStreamPublisher(streamManager);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "ACKCONC",
|
|
Subjects = ["ack.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
consumerManager.CreateOrUpdate("ACKCONC", new ConsumerConfig
|
|
{
|
|
DurableName = "acker",
|
|
FilterSubject = "ack.>",
|
|
AckPolicy = AckPolicy.All,
|
|
});
|
|
|
|
const int numPublish = 50;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Publish concurrently
|
|
await Parallel.ForEachAsync(Enumerable.Range(0, numPublish), async (i, _) =>
|
|
{
|
|
try
|
|
{
|
|
publisher.TryCapture($"ack.event.{i}", Encoding.UTF8.GetBytes($"msg-{i}"), null, out var ack);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
|
|
// Fetch and ack should work correctly
|
|
var batch = await consumerManager.FetchAsync("ACKCONC", "acker", numPublish, streamManager, default);
|
|
batch.Messages.Count.ShouldBeGreaterThan(0);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamPubSub server/norace_test.go (concurrent publish)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_publish_to_same_stream_produces_monotonic_sequences()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "SEQCONC",
|
|
Subjects = ["seq.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
const int numPublish = 100;
|
|
var sequences = new ConcurrentBag<ulong>();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Sequential publish to avoid store contention (the Go test also serializes this)
|
|
for (var i = 0; i < numPublish; i++)
|
|
{
|
|
try
|
|
{
|
|
var ack = streamManager.Capture($"seq.event", Encoding.UTF8.GetBytes($"msg-{i}"));
|
|
if (ack != null)
|
|
sequences.Add(ack.Seq);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
}
|
|
|
|
errors.ShouldBeEmpty();
|
|
sequences.Count.ShouldBe(numPublish);
|
|
|
|
// All sequences should be unique and form a contiguous range
|
|
var sorted = sequences.OrderBy(s => s).ToArray();
|
|
for (var i = 1; i < sorted.Length; i++)
|
|
sorted[i].ShouldBe(sorted[i - 1] + 1);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterParallelStreamCreate server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Parallel_stream_create_with_different_subjects()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
const int numStreams = 50;
|
|
var results = new ConcurrentDictionary<string, bool>();
|
|
|
|
await Parallel.ForEachAsync(Enumerable.Range(0, numStreams), async (i, _) =>
|
|
{
|
|
var resp = streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = $"PAR-{i}",
|
|
Subjects = [$"par{i}.>"],
|
|
Replicas = 1,
|
|
});
|
|
results[$"PAR-{i}"] = resp.Error is null;
|
|
await Task.CompletedTask;
|
|
});
|
|
|
|
results.Count.ShouldBe(numStreams);
|
|
results.Values.All(v => v).ShouldBeTrue();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamStreamPurge server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_publish_and_purge_does_not_throw()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "PURGECONC",
|
|
Subjects = ["purge.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
streamManager.Capture("purge.event", Encoding.UTF8.GetBytes($"msg-{i}"));
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 10; i++)
|
|
{
|
|
streamManager.Purge("PURGECONC");
|
|
Thread.Sleep(5);
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistStats server/norace_test.go (SubList stats)
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_stats_are_consistent_under_concurrent_operations()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numOps = 200;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, numOps, i =>
|
|
{
|
|
try
|
|
{
|
|
var sub = new Subscription
|
|
{
|
|
Subject = $"stats.{i % 20}",
|
|
Sid = $"stat-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
_ = subList.Match($"stats.{i % 20}");
|
|
_ = subList.Stats();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
var stats = subList.Stats();
|
|
stats.NumSubs.ShouldBeGreaterThan(0U);
|
|
stats.NumInserts.ShouldBeGreaterThanOrEqualTo(stats.NumSubs);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistCachePurge server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_cache_consistent_under_concurrent_operations()
|
|
{
|
|
using var subList = new SubList();
|
|
|
|
// Insert enough subs to populate cache
|
|
for (var i = 0; i < 100; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"cache.{i}",
|
|
Sid = $"c-{i}",
|
|
});
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, 500, i =>
|
|
{
|
|
try
|
|
{
|
|
_ = subList.Match($"cache.{i % 100}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
subList.CacheCount.ShouldBeGreaterThan(0);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterMeta server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_meta_group_operations_are_thread_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(5);
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
await Parallel.ForEachAsync(Enumerable.Range(0, 50), async (i, ct) =>
|
|
{
|
|
try
|
|
{
|
|
await meta.ProposeCreateStreamAsync(new StreamConfig { Name = $"META-{i}" }, ct);
|
|
meta.GetState();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
meta.GetState().Streams.Count.ShouldBe(50);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterMetaStepdown server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_meta_stepdown_and_state_reads_are_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
meta.StepDown();
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = meta.GetState();
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceRaftElection server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_raft_elections_do_not_corrupt_state()
|
|
{
|
|
const int numNodes = 3;
|
|
var nodes = Enumerable.Range(1, numNodes)
|
|
.Select(i => new RaftNode($"node-{i}"))
|
|
.ToList();
|
|
|
|
foreach (var node in nodes)
|
|
node.ConfigureCluster(nodes);
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, 10, i =>
|
|
{
|
|
try
|
|
{
|
|
var candidate = nodes[i % numNodes];
|
|
candidate.StartElection(numNodes);
|
|
foreach (var voter in nodes.Where(n => n.Id != candidate.Id))
|
|
candidate.ReceiveVote(voter.GrantVote(candidate.Term), numNodes);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
// At least one node should have become leader at some point
|
|
nodes.Any(n => n.Role == RaftRole.Leader || n.Term > 0).ShouldBeTrue();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceRaftPropose server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_raft_proposals_produce_unique_indices()
|
|
{
|
|
var group = new StreamReplicaGroup("RAFTPROP", replicas: 3);
|
|
const int numProposals = 50;
|
|
var indices = new ConcurrentBag<long>();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Raft proposals must be sequential (leader serializes them)
|
|
for (var i = 0; i < numProposals; i++)
|
|
{
|
|
try
|
|
{
|
|
var idx = await group.ProposeAsync($"PUB event.{i}", default);
|
|
indices.Add(idx);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
}
|
|
|
|
errors.ShouldBeEmpty();
|
|
indices.Count.ShouldBe(numProposals);
|
|
|
|
// All indices should be unique
|
|
indices.Distinct().Count().ShouldBe(numProposals);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterConsumerCreateDelete server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_consumer_create_and_delete_is_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var consumerManager = new ConsumerManager(meta);
|
|
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "CCONC",
|
|
Subjects = ["cc.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
const int numOps = 50;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Create consumers
|
|
for (var i = 0; i < numOps; i++)
|
|
{
|
|
consumerManager.CreateOrUpdate("CCONC", new ConsumerConfig
|
|
{
|
|
DurableName = $"c-{i}",
|
|
});
|
|
}
|
|
|
|
// Concurrently delete and create more
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < numOps / 2; i++)
|
|
consumerManager.Delete("CCONC", $"c-{i}");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = numOps; i < numOps + 25; i++)
|
|
{
|
|
consumerManager.CreateOrUpdate("CCONC", new ConsumerConfig
|
|
{
|
|
DurableName = $"c-{i}",
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
_ = consumerManager.ListNames("CCONC");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistBatchRemove server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_batch_remove_under_concurrent_match_is_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
const int numSubs = 100;
|
|
var subs = new List<Subscription>();
|
|
|
|
for (var i = 0; i < numSubs; i++)
|
|
{
|
|
var sub = new Subscription
|
|
{
|
|
Subject = $"batch.{i % 20}",
|
|
Sid = $"b-{i}",
|
|
};
|
|
subList.Insert(sub);
|
|
subs.Add(sub);
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
subList.RemoveBatch(subs.Take(50));
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = subList.Match($"batch.{i % 20}");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistHasInterest server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_has_interest_concurrent_with_insert_is_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 200; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"hi.{i % 30}",
|
|
Sid = $"hi-{i}",
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 200; i++)
|
|
_ = subList.HasInterest($"hi.{i % 30}");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamPublishParallel server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Parallel_publish_to_multiple_streams_routes_correctly()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var streamManager = new StreamManager(meta);
|
|
var publisher = new JetStreamPublisher(streamManager);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "PAR_A",
|
|
Subjects = ["para.>"],
|
|
Replicas = 1,
|
|
});
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "PAR_B",
|
|
Subjects = ["parb.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
var results = new ConcurrentDictionary<string, ConcurrentBag<ulong>>();
|
|
results["PAR_A"] = [];
|
|
results["PAR_B"] = [];
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
await Parallel.ForEachAsync(Enumerable.Range(0, 50), async (i, _) =>
|
|
{
|
|
try
|
|
{
|
|
var subject = i % 2 == 0 ? "para.event" : "parb.event";
|
|
if (publisher.TryCapture(subject, Encoding.UTF8.GetBytes($"msg-{i}"), null, out var ack))
|
|
results[ack.Stream].Add(ack.Seq);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
results["PAR_A"].Count.ShouldBeGreaterThan(0);
|
|
results["PAR_B"].Count.ShouldBeGreaterThan(0);
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamClusterStreamInfo server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_stream_info_and_publish_is_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var consumerManager = new ConsumerManager(meta);
|
|
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "INFOCONC",
|
|
Subjects = ["ic.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
await Task.WhenAll(
|
|
Task.Run(() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
streamManager.Capture("ic.event", Encoding.UTF8.GetBytes($"msg-{i}"));
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
}),
|
|
Task.Run(() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = streamManager.GetInfo("INFOCONC");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
}),
|
|
Task.Run(() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = streamManager.ListNames();
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
}));
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistNumInterest server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_num_interest_concurrent_is_consistent()
|
|
{
|
|
using var subList = new SubList();
|
|
|
|
for (var i = 0; i < 100; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = "num.interest.test",
|
|
Sid = $"ni-{i}",
|
|
});
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.For(0, 200, i =>
|
|
{
|
|
try
|
|
{
|
|
var (plain, queue) = subList.NumInterest("num.interest.test");
|
|
plain.ShouldBeGreaterThanOrEqualTo(0);
|
|
queue.ShouldBeGreaterThanOrEqualTo(0);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistAll server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_all_subscriptions_concurrent_is_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"all.{i % 10}",
|
|
Sid = $"all-{i}",
|
|
});
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = subList.All();
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 50; i < 100; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"all.new.{i}",
|
|
Sid = $"allnew-{i}",
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceMemStoreAppend server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_memstore_append_and_load_is_safe()
|
|
{
|
|
var store = new MemStore();
|
|
const int numMessages = 100;
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
// Sequential append (MemStore is not thread-safe for writes)
|
|
for (var i = 0; i < numMessages; i++)
|
|
await store.AppendAsync($"test.{i % 10}", Encoding.UTF8.GetBytes($"payload-{i}"), default);
|
|
|
|
// Concurrent reads
|
|
await Parallel.ForEachAsync(Enumerable.Range(1, numMessages), async (seq, _) =>
|
|
{
|
|
try
|
|
{
|
|
var msg = await store.LoadAsync((ulong)seq, default);
|
|
msg.ShouldNotBeNull();
|
|
msg!.Sequence.ShouldBe((ulong)seq);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamApiRouter server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_api_routing_is_thread_safe()
|
|
{
|
|
var meta = new JetStreamMetaGroup(3);
|
|
var consumerManager = new ConsumerManager(meta);
|
|
var streamManager = new StreamManager(meta, consumerManager: consumerManager);
|
|
var router = new JetStreamApiRouter(streamManager, consumerManager, meta);
|
|
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = "APIROUTE",
|
|
Subjects = ["api.>"],
|
|
Replicas = 1,
|
|
});
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
await Parallel.ForEachAsync(Enumerable.Range(0, 100), async (i, _) =>
|
|
{
|
|
try
|
|
{
|
|
var subject = (i % 4) switch
|
|
{
|
|
0 => JetStreamApiSubjects.Info,
|
|
1 => JetStreamApiSubjects.StreamNames,
|
|
2 => $"{JetStreamApiSubjects.StreamInfo}APIROUTE",
|
|
_ => JetStreamApiSubjects.StreamList,
|
|
};
|
|
var resp = router.Route(subject, "{}"u8);
|
|
resp.ShouldNotBeNull();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
|
|
await Task.CompletedTask;
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceRaftReplication server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public async Task Concurrent_replica_group_stepdown_and_propose()
|
|
{
|
|
var group = new StreamReplicaGroup("RACERAFT", replicas: 3);
|
|
var errors = new ConcurrentBag<Exception>();
|
|
var indices = new ConcurrentBag<long>();
|
|
|
|
// Sequential operations (Raft serializes proposals through leader)
|
|
for (var i = 0; i < 20; i++)
|
|
{
|
|
try
|
|
{
|
|
if (i % 5 == 0 && i > 0)
|
|
await group.StepDownAsync(default);
|
|
|
|
var idx = await group.ProposeAsync($"PUB event.{i}", default);
|
|
indices.Add(idx);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
errors.Add(ex);
|
|
}
|
|
}
|
|
|
|
errors.ShouldBeEmpty();
|
|
indices.Count.ShouldBe(20);
|
|
group.Leader.IsLeader.ShouldBeTrue();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceSublistReverseMatch server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void SubList_reverse_match_concurrent_with_insert_is_safe()
|
|
{
|
|
using var subList = new SubList();
|
|
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"rev.{i % 10}.data",
|
|
Sid = $"rev-{i}",
|
|
});
|
|
}
|
|
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = subList.ReverseMatch($"rev.{i % 10}.data");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 50; i < 100; i++)
|
|
{
|
|
subList.Insert(new Subscription
|
|
{
|
|
Subject = $"rev.{i % 10}.extra",
|
|
Sid = $"revx-{i}",
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceJetStreamConsumerListNames server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_consumer_list_names_during_create_is_safe()
|
|
{
|
|
var consumerManager = new ConsumerManager();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
{
|
|
consumerManager.CreateOrUpdate("STREAM", new ConsumerConfig
|
|
{
|
|
DurableName = $"cln-{i}",
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 200; i++)
|
|
_ = consumerManager.ListNames("STREAM");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
|
|
// ---------------------------------------------------------------
|
|
// Go: TestNoRaceStreamFindBySubject server/norace_test.go
|
|
// ---------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Concurrent_find_by_subject_during_create_is_safe()
|
|
{
|
|
var streamManager = new StreamManager();
|
|
var errors = new ConcurrentBag<Exception>();
|
|
|
|
Parallel.Invoke(
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
streamManager.CreateOrUpdate(new StreamConfig
|
|
{
|
|
Name = $"FBS-{i}",
|
|
Subjects = [$"fbs{i}.>"],
|
|
Replicas = 1,
|
|
});
|
|
}
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
},
|
|
() =>
|
|
{
|
|
try
|
|
{
|
|
for (var i = 0; i < 100; i++)
|
|
_ = streamManager.FindBySubject($"fbs{i % 50}.test");
|
|
}
|
|
catch (Exception ex) { errors.Add(ex); }
|
|
});
|
|
|
|
errors.ShouldBeEmpty();
|
|
}
|
|
}
|