Files
natsdotnet/tests/NATS.Server.Core.Tests/Stress/ConcurrentPubSubStressTests.cs
Joseph Doherty 7fbffffd05 refactor: rename remaining tests to NATS.Server.Core.Tests
- Rename tests/NATS.Server.Tests -> tests/NATS.Server.Core.Tests
- Update solution file, InternalsVisibleTo, and csproj references
- Remove JETSTREAM_INTEGRATION_MATRIX and NATS.NKeys from csproj (moved to JetStream.Tests and Auth.Tests)
- Update all namespaces from NATS.Server.Tests.* to NATS.Server.Core.Tests.*
- Replace private GetFreePort/ReadUntilAsync helpers with TestUtilities calls
- Fix stale namespace in Transport.Tests/NetworkingGoParityTests.cs
2026-03-12 16:14:02 -04:00

916 lines
29 KiB
C#

// Go parity: golang/nats-server/server/norace_1_test.go
// Covers: concurrent publish/subscribe thread safety, SubList trie integrity
// under high concurrency, wildcard routing under load, queue group balancing,
// cache invalidation safety, and subject tree concurrent insert/remove.
using System.Collections.Concurrent;
using NATS.Server.Subscriptions;
namespace NATS.Server.Core.Tests.Stress;
/// <summary>
/// Stress tests for concurrent pub/sub operations on the in-process SubList and SubjectMatch
/// classes. All tests use Parallel.For / Task.WhenAll to exercise thread safety directly
/// without spinning up a real NatsServer.
///
/// Go ref: norace_1_test.go — concurrent subscription and matching operations.
/// </summary>
public class ConcurrentPubSubStressTests
{
// ---------------------------------------------------------------
// Go: TestNoRaceSublistConcurrent100Subscribers norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_100_concurrent_subscribers_all_inserted_without_error()
{
// 100 concurrent goroutines each Subscribe to the same subject and then Match.
using var subList = new SubList();
const int count = 100;
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, count, i =>
{
try
{
subList.Insert(new Subscription { Subject = "stress.concurrent", Sid = $"s{i}" });
var result = subList.Match("stress.concurrent");
result.PlainSubs.Length.ShouldBeGreaterThan(0);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
subList.Count.ShouldBe((uint)count);
}
// ---------------------------------------------------------------
// Go: TestNoRace50ConcurrentPublishers norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_50_concurrent_publishers_produce_correct_match_counts()
{
// 50 goroutines each publish 100 times to their own subject.
// Verifies that Match never throws even under heavy concurrent write/read.
using var subList = new SubList();
const int publishers = 50;
const int messagesEach = 100;
var errors = new ConcurrentBag<Exception>();
// Pre-insert one subscription per publisher subject
for (var i = 0; i < publishers; i++)
{
subList.Insert(new Subscription
{
Subject = $"pub.stress.{i}",
Sid = $"pre-{i}",
});
}
Parallel.For(0, publishers, i =>
{
try
{
for (var j = 0; j < messagesEach; j++)
{
var result = subList.Match($"pub.stress.{i}");
result.PlainSubs.Length.ShouldBe(1);
}
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceSubUnsubConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_concurrent_subscribe_and_unsubscribe_does_not_crash()
{
using var subList = new SubList();
const int ops = 300;
var subs = new ConcurrentBag<Subscription>();
var errors = new ConcurrentBag<Exception>();
// Concurrent inserts and removes — neither side holds a reference the other
// side needs, so any interleaving is valid as long as it doesn't throw.
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < ops; i++)
{
var sub = new Subscription { Subject = $"unsub.{i % 30}", Sid = $"ins-{i}" };
subList.Insert(sub);
subs.Add(sub);
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
foreach (var sub in subs.Take(ops / 2))
subList.Remove(sub);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceConcurrentMatchOperations norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_concurrent_match_operations_are_thread_safe()
{
using var subList = new SubList();
for (var i = 0; i < 50; i++)
{
subList.Insert(new Subscription
{
Subject = $"match.safe.{i % 10}",
Sid = $"m{i}",
});
}
var errors = new ConcurrentBag<Exception>();
// 200 threads all calling Match simultaneously
Parallel.For(0, 200, i =>
{
try
{
var result = subList.Match($"match.safe.{i % 10}");
result.ShouldNotBeNull();
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRace1000ConcurrentSubscriptions norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_handles_1000_concurrent_subscriptions_without_error()
{
using var subList = new SubList();
const int count = 1000;
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, count, i =>
{
try
{
subList.Insert(new Subscription
{
Subject = $"big.load.{i % 100}",
Sid = $"big-{i}",
});
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
subList.Count.ShouldBe((uint)count);
}
// ---------------------------------------------------------------
// Go: TestNoRace10000SubscriptionsWithConcurrentMatch norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_handles_10000_subscriptions_with_concurrent_matches()
{
using var subList = new SubList();
const int count = 10_000;
// Sequential insert to avoid any write-write contention noise
for (var i = 0; i < count; i++)
{
subList.Insert(new Subscription
{
Subject = $"huge.{i % 200}.data",
Sid = $"h{i}",
});
}
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 500, i =>
{
try
{
var result = subList.Match($"huge.{i % 200}.data");
// Each subject bucket has count/200 = 50 subscribers
result.PlainSubs.Length.ShouldBe(50);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceWildcardConcurrentPub norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_wildcard_subjects_routed_correctly_under_concurrent_match()
{
using var subList = new SubList();
subList.Insert(new Subscription { Subject = "wc.*", Sid = "pwc" });
subList.Insert(new Subscription { Subject = "wc.>", Sid = "fwc" });
subList.Insert(new Subscription { Subject = "wc.specific", Sid = "lit" });
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 400, i =>
{
try
{
var subject = (i % 3) switch
{
0 => "wc.specific",
1 => "wc.anything",
_ => "wc.deep.nested",
};
var result = subList.Match(subject);
// wc.* matches single-token, wc.> matches all
result.PlainSubs.Length.ShouldBeGreaterThan(0);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceQueueGroupBalancingUnderLoad norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_queue_group_balancing_correct_under_concurrent_load()
{
using var subList = new SubList();
const int memberCount = 20;
for (var i = 0; i < memberCount; i++)
{
subList.Insert(new Subscription
{
Subject = "queue.load",
Queue = "workers",
Sid = $"q{i}",
});
}
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 200, i =>
{
try
{
var result = subList.Match("queue.load");
result.QueueSubs.Length.ShouldBe(1);
result.QueueSubs[0].Length.ShouldBe(memberCount);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRace100ConcurrentPubsSameSubject norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_100_concurrent_publishes_to_same_subject_all_processed()
{
using var subList = new SubList();
subList.Insert(new Subscription { Subject = "same.subject", Sid = "single" });
var matchCount = 0;
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 100, _ =>
{
try
{
var result = subList.Match("same.subject");
result.PlainSubs.Length.ShouldBe(1);
Interlocked.Increment(ref matchCount);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
matchCount.ShouldBe(100);
}
// ---------------------------------------------------------------
// Go: TestNoRaceConcurrentIdenticalSubjects norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_concurrent_subscribe_with_identical_subjects_all_inserted()
{
using var subList = new SubList();
const int count = 100;
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, count, i =>
{
try
{
subList.Insert(new Subscription
{
Subject = "identical.subject",
Sid = $"ident-{i}",
});
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
var result = subList.Match("identical.subject");
result.PlainSubs.Length.ShouldBe(count);
}
// ---------------------------------------------------------------
// Go: TestNoRaceSubscribePublishInterleaving norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_subscribe_publish_interleaving_does_not_lose_messages()
{
using var subList = new SubList();
var errors = new ConcurrentBag<Exception>();
var totalMatches = 0;
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < 100; i++)
{
subList.Insert(new Subscription
{
Subject = $"interleave.{i % 10}",
Sid = $"il-{i}",
});
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < 200; i++)
{
var result = subList.Match($"interleave.{i % 10}");
Interlocked.Add(ref totalMatches, result.PlainSubs.Length);
}
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
// We cannot assert a fixed count because of race between sub insert and match,
// but no exception is the primary invariant.
totalMatches.ShouldBeGreaterThanOrEqualTo(0);
}
// ---------------------------------------------------------------
// Go: TestNoRaceCacheInvalidationConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_cache_invalidation_is_thread_safe_under_concurrent_modifications()
{
using var subList = new SubList();
// Fill the cache
for (var i = 0; i < 100; i++)
{
var sub = new Subscription { Subject = $"cache.inv.{i}", Sid = $"ci-{i}" };
subList.Insert(sub);
_ = subList.Match($"cache.inv.{i}");
}
subList.CacheCount.ShouldBeGreaterThan(0);
var errors = new ConcurrentBag<Exception>();
// Concurrent reads (cache hits) and writes (cache invalidation)
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < 200; i++)
_ = subList.Match($"cache.inv.{i % 100}");
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 100; i < 150; i++)
{
subList.Insert(new Subscription
{
Subject = $"cache.inv.{i}",
Sid = $"cinew-{i}",
});
}
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRacePurgeAndMatchConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_concurrent_batch_remove_and_match_do_not_deadlock()
{
using var subList = new SubList();
var inserted = new List<Subscription>();
var errors = new ConcurrentBag<Exception>();
for (var i = 0; i < 200; i++)
{
var sub = new Subscription { Subject = $"purge.match.{i % 20}", Sid = $"pm-{i}" };
subList.Insert(sub);
inserted.Add(sub);
}
Parallel.Invoke(
() =>
{
try
{
subList.RemoveBatch(inserted.Take(100));
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < 100; i++)
_ = subList.Match($"purge.match.{i % 20}");
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRace1000Subjects10SubscribersEach norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_1000_subjects_10_subscribers_each_concurrent_match_correct()
{
using var subList = new SubList();
const int subjects = 200; // reduced for CI speed; same shape as 1000
const int subsPerSubject = 5;
for (var s = 0; s < subjects; s++)
{
for (var n = 0; n < subsPerSubject; n++)
{
subList.Insert(new Subscription
{
Subject = $"big.tree.{s}",
Sid = $"bt-{s}-{n}",
});
}
}
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, subjects * 3, i =>
{
try
{
var result = subList.Match($"big.tree.{i % subjects}");
result.PlainSubs.Length.ShouldBe(subsPerSubject);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceMixedWildcardLiteralConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_mixed_wildcard_and_literal_subscriptions_under_concurrent_match()
{
using var subList = new SubList();
// Mix of literals, * wildcards, and > wildcards
for (var i = 0; i < 20; i++)
{
subList.Insert(new Subscription { Subject = $"mix.{i}.literal", Sid = $"lit-{i}" });
subList.Insert(new Subscription { Subject = $"mix.{i}.*", Sid = $"pwc-{i}" });
}
subList.Insert(new Subscription { Subject = "mix.>", Sid = "fwc-root" });
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 300, i =>
{
try
{
var idx = i % 20;
var result = subList.Match($"mix.{idx}.literal");
// Matches: the literal sub, the * wildcard sub, and the > sub
result.PlainSubs.Length.ShouldBe(3);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceHighThroughputPublish norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_high_throughput_10000_messages_to_single_subscriber()
{
using var subList = new SubList();
subList.Insert(new Subscription { Subject = "throughput.test", Sid = "tp1" });
var count = 0;
var errors = new ConcurrentBag<Exception>();
for (var i = 0; i < 10_000; i++)
{
try
{
var result = subList.Match("throughput.test");
result.PlainSubs.Length.ShouldBe(1);
count++;
}
catch (Exception ex) { errors.Add(ex); }
}
errors.ShouldBeEmpty();
count.ShouldBe(10_000);
}
// ---------------------------------------------------------------
// Go: TestNoRaceQueueSubConcurrentUnsubscribe norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_concurrent_queue_group_subscribe_and_unsubscribe_is_safe()
{
using var subList = new SubList();
const int ops = 200;
var inserted = new ConcurrentBag<Subscription>();
var errors = new ConcurrentBag<Exception>();
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < ops; i++)
{
var sub = new Subscription
{
Subject = $"qg.stress.{i % 10}",
Queue = $"grp-{i % 5}",
Sid = $"qgs-{i}",
};
subList.Insert(sub);
inserted.Add(sub);
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
foreach (var sub in inserted.Take(ops / 2))
subList.Remove(sub);
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < ops; i++)
_ = subList.Match($"qg.stress.{i % 10}");
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRace500Subjects5SubscribersEach norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_500_subjects_5_subscribers_each_concurrent_match_returns_correct_results()
{
using var subList = new SubList();
const int subjects = 100; // scaled for CI speed
const int subsPerSubject = 5;
for (var s = 0; s < subjects; s++)
{
for (var n = 0; n < subsPerSubject; n++)
{
subList.Insert(new Subscription
{
Subject = $"five.subs.{s}",
Sid = $"fs-{s}-{n}",
});
}
}
var errors = new ConcurrentBag<Exception>();
var correctCount = 0;
Parallel.For(0, subjects * 4, i =>
{
try
{
var result = subList.Match($"five.subs.{i % subjects}");
if (result.PlainSubs.Length == subsPerSubject)
Interlocked.Increment(ref correctCount);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
correctCount.ShouldBe(subjects * 4);
}
// ---------------------------------------------------------------
// Go: TestNoRaceSubjectValidationConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubjectMatch_validation_is_thread_safe_under_concurrent_calls()
{
var errors = new ConcurrentBag<Exception>();
var validCount = 0;
Parallel.For(0, 1000, i =>
{
try
{
var subject = (i % 4) switch
{
0 => $"valid.subject.{i}",
1 => $"valid.*.wildcard",
2 => $"valid.>",
_ => string.Empty, // invalid
};
var isValid = SubjectMatch.IsValidSubject(subject);
if (isValid)
Interlocked.Increment(ref validCount);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
// 750 valid, 250 empty (invalid)
validCount.ShouldBe(750);
}
// ---------------------------------------------------------------
// Go: TestNoRaceHasInterestConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_has_interest_returns_consistent_results_under_concurrent_insert()
{
using var subList = new SubList();
var errors = new ConcurrentBag<Exception>();
var interestFoundCount = 0;
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < 200; i++)
{
subList.Insert(new Subscription
{
Subject = $"interest.{i % 20}",
Sid = $"hi-{i}",
});
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < 200; i++)
{
if (subList.HasInterest($"interest.{i % 20}"))
Interlocked.Increment(ref interestFoundCount);
}
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
interestFoundCount.ShouldBeGreaterThanOrEqualTo(0);
}
// ---------------------------------------------------------------
// Go: TestNoRaceNumInterestConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_num_interest_is_consistent_under_high_concurrency()
{
using var subList = new SubList();
const int subCount = 80;
for (var i = 0; i < subCount; i++)
{
subList.Insert(new Subscription
{
Subject = "num.interest.stress",
Sid = $"nis-{i}",
});
}
var errors = new ConcurrentBag<Exception>();
Parallel.For(0, 400, _ =>
{
try
{
var (plain, queue) = subList.NumInterest("num.interest.stress");
plain.ShouldBe(subCount);
queue.ShouldBe(0);
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceReverseMatchConcurrent norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_reverse_match_concurrent_with_inserts_does_not_throw()
{
using var subList = new SubList();
var errors = new ConcurrentBag<Exception>();
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < 100; i++)
{
subList.Insert(new Subscription
{
Subject = $"rev.stress.{i % 10}",
Sid = $"rs-{i}",
});
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < 150; i++)
_ = subList.ReverseMatch($"rev.stress.{i % 10}");
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
}
// ---------------------------------------------------------------
// Go: TestNoRaceStatsConsistencyUnderLoad norace_1_test.go
// ---------------------------------------------------------------
[Fact]
[Trait("Category", "Stress")]
public void SubList_stats_remain_consistent_under_concurrent_insert_remove_match()
{
using var subList = new SubList();
const int ops = 300;
var insertedSubs = new ConcurrentBag<Subscription>();
var errors = new ConcurrentBag<Exception>();
Parallel.Invoke(
() =>
{
try
{
for (var i = 0; i < ops; i++)
{
var sub = new Subscription
{
Subject = $"stats.stress.{i % 30}",
Sid = $"ss-{i}",
};
subList.Insert(sub);
insertedSubs.Add(sub);
}
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < ops; i++)
_ = subList.Match($"stats.stress.{i % 30}");
}
catch (Exception ex) { errors.Add(ex); }
},
() =>
{
try
{
for (var i = 0; i < 50; i++)
_ = subList.Stats();
}
catch (Exception ex) { errors.Add(ex); }
});
errors.ShouldBeEmpty();
var finalStats = subList.Stats();
finalStats.NumInserts.ShouldBeGreaterThan(0UL);
finalStats.NumMatches.ShouldBeGreaterThan(0UL);
}
}