Fix E2E test gaps and add comprehensive E2E + parity test suites
- Fix pull consumer fetch: send original stream subject in HMSG (not inbox) so NATS client distinguishes data messages from control messages - Fix MaxAge expiry: add background timer in StreamManager for periodic pruning - Fix JetStream wire format: Go-compatible anonymous objects with string enums, proper offset-based pagination for stream/consumer list APIs - Add 42 E2E black-box tests (core messaging, auth, TLS, accounts, JetStream) - Add ~1000 parity tests across all subsystems (gaps closure) - Update gap inventory docs to reflect implementation status
This commit is contained in:
@@ -26,11 +26,77 @@ public sealed class SubList : IDisposable
|
||||
private ulong _inserts;
|
||||
private ulong _removes;
|
||||
private int _highFanoutNodes;
|
||||
private Action<bool>? _interestStateNotification;
|
||||
private readonly Dictionary<string, List<Action<bool>>> _queueInsertNotifications = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<Action<bool>>> _queueRemoveNotifications = new(StringComparer.Ordinal);
|
||||
|
||||
private readonly record struct CachedResult(SubListResult Result, long Generation);
|
||||
internal readonly record struct RoutedSubKeyInfo(string RouteId, string Account, string Subject, string? Queue);
|
||||
|
||||
public event Action<InterestChange>? InterestChanged;
|
||||
|
||||
public SubList()
|
||||
: this(enableCache: true)
|
||||
{
|
||||
}
|
||||
|
||||
public SubList(bool enableCache)
|
||||
{
|
||||
if (!enableCache)
|
||||
_cache = null;
|
||||
}
|
||||
|
||||
public static SubList NewSublistNoCache() => new(enableCache: false);
|
||||
|
||||
public bool CacheEnabled() => _cache != null;
|
||||
|
||||
public void RegisterNotification(Action<bool> callback) => _interestStateNotification = callback;
|
||||
|
||||
public void ClearNotification() => _interestStateNotification = null;
|
||||
|
||||
public bool RegisterQueueNotification(string subject, string queue, Action<bool> callback)
|
||||
{
|
||||
if (callback == null || string.IsNullOrWhiteSpace(subject) || string.IsNullOrWhiteSpace(queue))
|
||||
return false;
|
||||
if (SubjectMatch.SubjectHasWildcard(subject) || SubjectMatch.SubjectHasWildcard(queue))
|
||||
return false;
|
||||
|
||||
bool hasInterest;
|
||||
var key = QueueNotifyKey(subject, queue);
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
hasInterest = HasExactQueueInterestNoLock(subject, queue);
|
||||
var map = hasInterest ? _queueRemoveNotifications : _queueInsertNotifications;
|
||||
if (!AddQueueNotifyNoLock(map, key, callback))
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
|
||||
callback(hasInterest);
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool ClearQueueNotification(string subject, string queue, Action<bool> callback)
|
||||
{
|
||||
var key = QueueNotifyKey(subject, queue);
|
||||
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var removed = RemoveQueueNotifyNoLock(_queueRemoveNotifications, key, callback);
|
||||
removed |= RemoveQueueNotifyNoLock(_queueInsertNotifications, key, callback);
|
||||
return removed;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
@@ -112,7 +178,7 @@ public sealed class SubList : IDisposable
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var key = $"{sub.RouteId}|{sub.Account}|{sub.Subject}|{sub.Queue}";
|
||||
var key = BuildRoutedSubKey(sub.RouteId, sub.Account, sub.Subject, sub.Queue);
|
||||
var changed = false;
|
||||
if (sub.IsRemoval)
|
||||
{
|
||||
@@ -149,6 +215,127 @@ public sealed class SubList : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
public void UpdateRemoteQSub(RemoteSubscription sub)
|
||||
{
|
||||
if (sub.Queue == null)
|
||||
return;
|
||||
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var key = BuildRoutedSubKey(sub.RouteId, sub.Account, sub.Subject, sub.Queue);
|
||||
if (!_remoteSubs.TryGetValue(key, out var existing))
|
||||
return;
|
||||
|
||||
var nextWeight = Math.Max(1, sub.QueueWeight);
|
||||
if (existing.QueueWeight == nextWeight)
|
||||
return;
|
||||
|
||||
_remoteSubs[key] = existing with { QueueWeight = nextWeight };
|
||||
Interlocked.Increment(ref _generation);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
internal static string BuildRoutedSubKey(string routeId, string account, string subject, string? queue)
|
||||
=> $"{routeId}|{account}|{subject}|{queue}";
|
||||
|
||||
internal static string? GetAccNameFromRoutedSubKey(string routedSubKey)
|
||||
=> GetRoutedSubKeyInfo(routedSubKey)?.Account;
|
||||
|
||||
internal static RoutedSubKeyInfo? GetRoutedSubKeyInfo(string routedSubKey)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(routedSubKey))
|
||||
return null;
|
||||
|
||||
var parts = routedSubKey.Split('|');
|
||||
if (parts.Length != 4)
|
||||
return null;
|
||||
|
||||
if (parts[0].Length == 0 || parts[1].Length == 0 || parts[2].Length == 0)
|
||||
return null;
|
||||
|
||||
var queue = parts[3].Length == 0 ? null : parts[3];
|
||||
return new RoutedSubKeyInfo(parts[0], parts[1], parts[2], queue);
|
||||
}
|
||||
|
||||
public int RemoveRemoteSubs(string routeId)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var removed = 0;
|
||||
foreach (var kvp in _remoteSubs.ToArray())
|
||||
{
|
||||
var info = GetRoutedSubKeyInfo(kvp.Key);
|
||||
if (info == null || !string.Equals(info.Value.RouteId, routeId, StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
if (_remoteSubs.Remove(kvp.Key))
|
||||
{
|
||||
removed++;
|
||||
InterestChanged?.Invoke(new InterestChange(
|
||||
InterestChangeKind.RemoteRemoved,
|
||||
kvp.Value.Subject,
|
||||
kvp.Value.Queue,
|
||||
kvp.Value.Account));
|
||||
}
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
Interlocked.Increment(ref _generation);
|
||||
|
||||
return removed;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
public int RemoveRemoteSubsForAccount(string routeId, string account)
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var removed = 0;
|
||||
foreach (var kvp in _remoteSubs.ToArray())
|
||||
{
|
||||
var info = GetRoutedSubKeyInfo(kvp.Key);
|
||||
if (info == null)
|
||||
continue;
|
||||
|
||||
if (!string.Equals(info.Value.RouteId, routeId, StringComparison.Ordinal)
|
||||
|| !string.Equals(info.Value.Account, account, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_remoteSubs.Remove(kvp.Key))
|
||||
{
|
||||
removed++;
|
||||
InterestChanged?.Invoke(new InterestChange(
|
||||
InterestChangeKind.RemoteRemoved,
|
||||
kvp.Value.Subject,
|
||||
kvp.Value.Queue,
|
||||
kvp.Value.Account));
|
||||
}
|
||||
}
|
||||
|
||||
if (removed > 0)
|
||||
Interlocked.Increment(ref _generation);
|
||||
|
||||
return removed;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitWriteLock();
|
||||
}
|
||||
}
|
||||
|
||||
public bool HasRemoteInterest(string subject)
|
||||
=> HasRemoteInterest("$G", subject);
|
||||
|
||||
@@ -183,6 +370,7 @@ public sealed class SubList : IDisposable
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var hadInterest = _count > 0;
|
||||
var level = _root;
|
||||
TrieNode? node = null;
|
||||
bool sawFwc = false;
|
||||
@@ -240,6 +428,10 @@ public sealed class SubList : IDisposable
|
||||
_count++;
|
||||
_inserts++;
|
||||
Interlocked.Increment(ref _generation);
|
||||
if (sub.Queue != null && _queueInsertNotifications.Count > 0)
|
||||
CheckForQueueInsertNotificationNoLock(sub.Subject, sub.Queue);
|
||||
if (!hadInterest && _count > 0)
|
||||
_interestStateNotification?.Invoke(true);
|
||||
InterestChanged?.Invoke(new InterestChange(
|
||||
InterestChangeKind.LocalAdded,
|
||||
sub.Subject,
|
||||
@@ -258,10 +450,15 @@ public sealed class SubList : IDisposable
|
||||
_lock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
var hadInterest = _count > 0;
|
||||
if (RemoveInternal(sub))
|
||||
{
|
||||
_removes++;
|
||||
Interlocked.Increment(ref _generation);
|
||||
if (sub.Queue != null && _queueRemoveNotifications.Count > 0)
|
||||
CheckForQueueRemoveNotificationNoLock(sub.Subject, sub.Queue);
|
||||
if (hadInterest && _count == 0)
|
||||
_interestStateNotification?.Invoke(false);
|
||||
InterestChanged?.Invoke(new InterestChange(
|
||||
InterestChangeKind.LocalRemoved,
|
||||
sub.Subject,
|
||||
@@ -455,6 +652,90 @@ public sealed class SubList : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private static string QueueNotifyKey(string subject, string queue) => $"{subject} {queue}";
|
||||
|
||||
private static bool AddQueueNotifyNoLock(Dictionary<string, List<Action<bool>>> map, string key, Action<bool> callback)
|
||||
{
|
||||
if (!map.TryGetValue(key, out var callbacks))
|
||||
{
|
||||
callbacks = [];
|
||||
map[key] = callbacks;
|
||||
}
|
||||
else if (callbacks.Contains(callback))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
callbacks.Add(callback);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool RemoveQueueNotifyNoLock(Dictionary<string, List<Action<bool>>> map, string key, Action<bool> callback)
|
||||
{
|
||||
if (!map.TryGetValue(key, out var callbacks))
|
||||
return false;
|
||||
|
||||
var removed = callbacks.Remove(callback);
|
||||
if (callbacks.Count == 0)
|
||||
map.Remove(key);
|
||||
return removed;
|
||||
}
|
||||
|
||||
private bool HasExactQueueInterestNoLock(string subject, string queue)
|
||||
{
|
||||
var subs = new List<Subscription>();
|
||||
CollectAll(_root, subs);
|
||||
foreach (var sub in subs)
|
||||
{
|
||||
if (sub.Queue != null
|
||||
&& string.Equals(sub.Subject, subject, StringComparison.Ordinal)
|
||||
&& string.Equals(sub.Queue, queue, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private void CheckForQueueInsertNotificationNoLock(string subject, string queue)
|
||||
{
|
||||
var key = QueueNotifyKey(subject, queue);
|
||||
if (!_queueInsertNotifications.TryGetValue(key, out var callbacks) || callbacks.Count == 0)
|
||||
return;
|
||||
|
||||
foreach (var callback in callbacks)
|
||||
callback(true);
|
||||
|
||||
if (!_queueRemoveNotifications.TryGetValue(key, out var removeCallbacks))
|
||||
{
|
||||
removeCallbacks = [];
|
||||
_queueRemoveNotifications[key] = removeCallbacks;
|
||||
}
|
||||
removeCallbacks.AddRange(callbacks);
|
||||
_queueInsertNotifications.Remove(key);
|
||||
}
|
||||
|
||||
private void CheckForQueueRemoveNotificationNoLock(string subject, string queue)
|
||||
{
|
||||
var key = QueueNotifyKey(subject, queue);
|
||||
if (!_queueRemoveNotifications.TryGetValue(key, out var callbacks) || callbacks.Count == 0)
|
||||
return;
|
||||
if (HasExactQueueInterestNoLock(subject, queue))
|
||||
return;
|
||||
|
||||
foreach (var callback in callbacks)
|
||||
callback(false);
|
||||
|
||||
if (!_queueInsertNotifications.TryGetValue(key, out var insertCallbacks))
|
||||
{
|
||||
insertCallbacks = [];
|
||||
_queueInsertNotifications[key] = insertCallbacks;
|
||||
}
|
||||
insertCallbacks.AddRange(callbacks);
|
||||
_queueRemoveNotifications.Remove(key);
|
||||
}
|
||||
|
||||
private void SweepCache()
|
||||
{
|
||||
_lock.EnterWriteLock();
|
||||
@@ -632,6 +913,9 @@ public sealed class SubList : IDisposable
|
||||
CacheHitRate = hitRate,
|
||||
MaxFanout = maxFanout,
|
||||
AvgFanout = cacheEntries > 0 ? (double)totalFanout / cacheEntries : 0.0,
|
||||
TotalFanout = (int)totalFanout,
|
||||
CacheEntries = cacheEntries,
|
||||
CacheHits = cacheHits,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -695,7 +979,11 @@ public sealed class SubList : IDisposable
|
||||
foreach (var sub in subs)
|
||||
{
|
||||
if (RemoveInternal(sub))
|
||||
{
|
||||
_removes++;
|
||||
if (sub.Queue != null && _queueRemoveNotifications.Count > 0)
|
||||
CheckForQueueRemoveNotificationNoLock(sub.Subject, sub.Queue);
|
||||
}
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref _generation);
|
||||
@@ -724,6 +1012,34 @@ public sealed class SubList : IDisposable
|
||||
return subs;
|
||||
}
|
||||
|
||||
public IReadOnlyList<Subscription> LocalSubs(bool includeLeafHubs = false)
|
||||
{
|
||||
var subs = new List<Subscription>();
|
||||
_lock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
CollectLocalSubs(_root, subs, includeLeafHubs);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitReadLock();
|
||||
}
|
||||
return subs;
|
||||
}
|
||||
|
||||
internal int NumLevels()
|
||||
{
|
||||
_lock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
return VisitLevel(_root, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
public SubListResult ReverseMatch(string subject)
|
||||
{
|
||||
var tokens = Tokenize(subject);
|
||||
@@ -857,6 +1173,82 @@ public sealed class SubList : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private static void CollectLocalSubs(TrieLevel level, List<Subscription> subs, bool includeLeafHubs)
|
||||
{
|
||||
foreach (var (_, node) in level.Nodes)
|
||||
{
|
||||
AddNodeLocalSubs(node, subs, includeLeafHubs);
|
||||
if (node.Next != null)
|
||||
CollectLocalSubs(node.Next, subs, includeLeafHubs);
|
||||
}
|
||||
if (level.Pwc != null)
|
||||
{
|
||||
AddNodeLocalSubs(level.Pwc, subs, includeLeafHubs);
|
||||
if (level.Pwc.Next != null)
|
||||
CollectLocalSubs(level.Pwc.Next, subs, includeLeafHubs);
|
||||
}
|
||||
if (level.Fwc != null)
|
||||
{
|
||||
AddNodeLocalSubs(level.Fwc, subs, includeLeafHubs);
|
||||
if (level.Fwc.Next != null)
|
||||
CollectLocalSubs(level.Fwc.Next, subs, includeLeafHubs);
|
||||
}
|
||||
}
|
||||
|
||||
private static void AddNodeLocalSubs(TrieNode node, List<Subscription> subs, bool includeLeafHubs)
|
||||
{
|
||||
foreach (var sub in node.PlainSubs)
|
||||
AddLocalSub(sub, subs, includeLeafHubs);
|
||||
foreach (var (_, qset) in node.QueueSubs)
|
||||
foreach (var sub in qset)
|
||||
AddLocalSub(sub, subs, includeLeafHubs);
|
||||
}
|
||||
|
||||
private static void AddLocalSub(Subscription sub, List<Subscription> subs, bool includeLeafHubs)
|
||||
{
|
||||
if (sub.Client == null)
|
||||
return;
|
||||
|
||||
var kind = sub.Client.Kind;
|
||||
if (kind is global::NATS.Server.ClientKind.Client
|
||||
or global::NATS.Server.ClientKind.System
|
||||
or global::NATS.Server.ClientKind.JetStream
|
||||
or global::NATS.Server.ClientKind.Account
|
||||
|| (includeLeafHubs && kind == global::NATS.Server.ClientKind.Leaf))
|
||||
{
|
||||
subs.Add(sub);
|
||||
}
|
||||
}
|
||||
|
||||
private static int VisitLevel(TrieLevel? level, int depth)
|
||||
{
|
||||
if (level == null || (level.Nodes.Count == 0 && level.Pwc == null && level.Fwc == null))
|
||||
return depth;
|
||||
|
||||
depth++;
|
||||
var maxDepth = depth;
|
||||
foreach (var (_, node) in level.Nodes)
|
||||
{
|
||||
var childDepth = VisitLevel(node.Next, depth);
|
||||
if (childDepth > maxDepth)
|
||||
maxDepth = childDepth;
|
||||
}
|
||||
if (level.Pwc != null)
|
||||
{
|
||||
var pwcDepth = VisitLevel(level.Pwc.Next, depth);
|
||||
if (pwcDepth > maxDepth)
|
||||
maxDepth = pwcDepth;
|
||||
}
|
||||
if (level.Fwc != null)
|
||||
{
|
||||
var fwcDepth = VisitLevel(level.Fwc.Next, depth);
|
||||
if (fwcDepth > maxDepth)
|
||||
maxDepth = fwcDepth;
|
||||
}
|
||||
|
||||
return maxDepth;
|
||||
}
|
||||
|
||||
private static void ReverseMatchLevel(TrieLevel? level, string[] tokens, int tokenIndex,
|
||||
List<Subscription> plainSubs, List<List<Subscription>> queueSubs)
|
||||
{
|
||||
|
||||
@@ -2,12 +2,36 @@ namespace NATS.Server.Subscriptions;
|
||||
|
||||
public sealed class SubListStats
|
||||
{
|
||||
public uint NumSubs { get; init; }
|
||||
public uint NumCache { get; init; }
|
||||
public ulong NumInserts { get; init; }
|
||||
public ulong NumRemoves { get; init; }
|
||||
public ulong NumMatches { get; init; }
|
||||
public double CacheHitRate { get; init; }
|
||||
public uint MaxFanout { get; init; }
|
||||
public double AvgFanout { get; init; }
|
||||
public uint NumSubs { get; set; }
|
||||
public uint NumCache { get; set; }
|
||||
public ulong NumInserts { get; set; }
|
||||
public ulong NumRemoves { get; set; }
|
||||
public ulong NumMatches { get; set; }
|
||||
public double CacheHitRate { get; set; }
|
||||
public uint MaxFanout { get; set; }
|
||||
public double AvgFanout { get; set; }
|
||||
|
||||
internal int TotalFanout { get; set; }
|
||||
internal int CacheEntries { get; set; }
|
||||
internal ulong CacheHits { get; set; }
|
||||
|
||||
public void Add(SubListStats stat)
|
||||
{
|
||||
NumSubs += stat.NumSubs;
|
||||
NumCache += stat.NumCache;
|
||||
NumInserts += stat.NumInserts;
|
||||
NumRemoves += stat.NumRemoves;
|
||||
NumMatches += stat.NumMatches;
|
||||
CacheHits += stat.CacheHits;
|
||||
if (MaxFanout < stat.MaxFanout)
|
||||
MaxFanout = stat.MaxFanout;
|
||||
|
||||
TotalFanout += stat.TotalFanout;
|
||||
CacheEntries += stat.CacheEntries;
|
||||
if (TotalFanout > 0 && CacheEntries > 0)
|
||||
AvgFanout = (double)TotalFanout / CacheEntries;
|
||||
|
||||
if (NumMatches > 0)
|
||||
CacheHitRate = (double)CacheHits / NumMatches;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +69,10 @@ public static class SubjectMatch
|
||||
return IsValidSubject(subject) && IsLiteral(subject);
|
||||
}
|
||||
|
||||
public static bool SubjectHasWildcard(string subject) => !IsLiteral(subject);
|
||||
|
||||
public static bool IsValidLiteralSubject(string subject) => IsValidPublishSubject(subject);
|
||||
|
||||
/// <summary>
|
||||
/// Match a literal subject against a pattern that may contain wildcards.
|
||||
/// </summary>
|
||||
@@ -196,6 +200,55 @@ public static class SubjectMatch
|
||||
return true;
|
||||
}
|
||||
|
||||
// Go reference: sublist.go SubjectMatchesFilter / subjectIsSubsetMatch / isSubsetMatch / isSubsetMatchTokenized.
|
||||
// This is used by JetStream stores to evaluate subject filters with wildcard semantics.
|
||||
public static bool SubjectMatchesFilter(string subject, string filter) => SubjectIsSubsetMatch(subject, filter);
|
||||
|
||||
public static bool SubjectIsSubsetMatch(string subject, string test)
|
||||
{
|
||||
var subjectTokens = TokenizeSubject(subject);
|
||||
return IsSubsetMatch(subjectTokens, test);
|
||||
}
|
||||
|
||||
public static bool IsSubsetMatch(string[] tokens, string test)
|
||||
{
|
||||
var testTokens = TokenizeSubject(test);
|
||||
return IsSubsetMatchTokenized(tokens, testTokens);
|
||||
}
|
||||
|
||||
public static bool IsSubsetMatchTokenized(IReadOnlyList<string> tokens, IReadOnlyList<string> test)
|
||||
{
|
||||
for (var i = 0; i < test.Count; i++)
|
||||
{
|
||||
if (i >= tokens.Count)
|
||||
return false;
|
||||
|
||||
var t2 = test[i];
|
||||
if (t2.Length == 0)
|
||||
return false;
|
||||
|
||||
if (t2.Length == 1 && t2[0] == Fwc)
|
||||
return true;
|
||||
|
||||
var t1 = tokens[i];
|
||||
if (t1.Length == 0 || (t1.Length == 1 && t1[0] == Fwc))
|
||||
return false;
|
||||
|
||||
if (t1.Length == 1 && t1[0] == Pwc)
|
||||
{
|
||||
var bothPwc = t2.Length == 1 && t2[0] == Pwc;
|
||||
if (!bothPwc)
|
||||
return false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(t2.Length == 1 && t2[0] == Pwc) && !string.Equals(t1, t2, StringComparison.Ordinal))
|
||||
return false;
|
||||
}
|
||||
|
||||
return tokens.Count == test.Count;
|
||||
}
|
||||
|
||||
private static bool TokensCanMatch(ReadOnlySpan<char> t1, ReadOnlySpan<char> t2)
|
||||
{
|
||||
if (t1.Length == 1 && (t1[0] == Pwc || t1[0] == Fwc))
|
||||
@@ -205,6 +258,8 @@ public static class SubjectMatch
|
||||
return t1.SequenceEqual(t2);
|
||||
}
|
||||
|
||||
private static string[] TokenizeSubject(string subject) => (subject ?? string.Empty).Split(Sep);
|
||||
|
||||
/// <summary>
|
||||
/// Validates subject. When checkRunes is true, also rejects null bytes.
|
||||
/// </summary>
|
||||
|
||||
@@ -108,6 +108,10 @@ public sealed partial class SubjectTransform
|
||||
{
|
||||
ops[i] = new TransformOp(TransformType.Partition, [], parsed.IntArg, parsed.StringArg);
|
||||
}
|
||||
else if (parsed.Type == TransformType.Random)
|
||||
{
|
||||
ops[i] = new TransformOp(TransformType.Random, [], parsed.IntArg, parsed.StringArg);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Other functions not allowed without wildcards in source
|
||||
@@ -119,6 +123,109 @@ public sealed partial class SubjectTransform
|
||||
return new SubjectTransform(source, destination, srcTokens, destTokens, ops);
|
||||
}
|
||||
|
||||
public static SubjectTransform? NewSubjectTransformWithStrict(string source, string destination, bool strict)
|
||||
{
|
||||
var transform = Create(source, destination);
|
||||
if (transform == null || !strict)
|
||||
return transform;
|
||||
|
||||
return UsesAllSourceWildcards(source, destination) ? transform : null;
|
||||
}
|
||||
|
||||
public static SubjectTransform? NewSubjectTransformStrict(string source, string destination)
|
||||
=> NewSubjectTransformWithStrict(source, destination, strict: true);
|
||||
|
||||
public static bool ValidateMapping(string destination)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(destination))
|
||||
return false;
|
||||
|
||||
var (valid, tokens, pwcCount, _) = SubjectInfo(destination);
|
||||
if (!valid || pwcCount > 0)
|
||||
return false;
|
||||
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
if (ParseDestToken(token) == null)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public static string TransformTokenize(string subject)
|
||||
{
|
||||
if (string.IsNullOrEmpty(subject))
|
||||
return subject;
|
||||
|
||||
var tokens = subject.Split('.');
|
||||
var wildcard = 0;
|
||||
|
||||
for (var i = 0; i < tokens.Length; i++)
|
||||
{
|
||||
if (tokens[i] == "*")
|
||||
tokens[i] = $"${++wildcard}";
|
||||
}
|
||||
|
||||
return string.Join('.', tokens);
|
||||
}
|
||||
|
||||
public static string TransformUntokenize(string subject)
|
||||
{
|
||||
if (string.IsNullOrEmpty(subject))
|
||||
return subject;
|
||||
|
||||
var tokens = subject.Split('.');
|
||||
for (var i = 0; i < tokens.Length; i++)
|
||||
{
|
||||
if (TryParseWildcardToken(tokens[i], out _))
|
||||
tokens[i] = "*";
|
||||
}
|
||||
|
||||
return string.Join('.', tokens);
|
||||
}
|
||||
|
||||
public SubjectTransform? Reverse()
|
||||
{
|
||||
var tokenizedSource = TransformTokenize(_source);
|
||||
var tokenizedDest = TransformTokenize(_dest);
|
||||
|
||||
var sourceTokens = tokenizedSource.Split('.');
|
||||
var destTokens = tokenizedDest.Split('.');
|
||||
|
||||
var oldToNewWildcard = new Dictionary<int, int>();
|
||||
var reverseWildcard = 0;
|
||||
foreach (var token in destTokens)
|
||||
{
|
||||
if (!TryParseWildcardToken(token, out var sourceWildcard))
|
||||
continue;
|
||||
|
||||
reverseWildcard++;
|
||||
if (!oldToNewWildcard.ContainsKey(sourceWildcard))
|
||||
oldToNewWildcard[sourceWildcard] = reverseWildcard;
|
||||
}
|
||||
|
||||
var reverseDestTokens = new string[sourceTokens.Length];
|
||||
for (var i = 0; i < sourceTokens.Length; i++)
|
||||
{
|
||||
var token = sourceTokens[i];
|
||||
if (!TryParseWildcardToken(token, out var sourceWildcard))
|
||||
{
|
||||
reverseDestTokens[i] = token;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!oldToNewWildcard.TryGetValue(sourceWildcard, out var mappedWildcard))
|
||||
return null;
|
||||
|
||||
reverseDestTokens[i] = $"${mappedWildcard}";
|
||||
}
|
||||
|
||||
var reverseSource = TransformUntokenize(tokenizedDest);
|
||||
var reverseDest = string.Join('.', reverseDestTokens);
|
||||
return Create(reverseSource, reverseDest);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Matches subject against source pattern, captures wildcard values, evaluates destination template.
|
||||
/// Returns null if subject doesn't match source.
|
||||
@@ -141,6 +248,14 @@ public sealed partial class SubjectTransform
|
||||
return TransformTokenized(subjectTokens);
|
||||
}
|
||||
|
||||
public string TransformSubject(string subject)
|
||||
{
|
||||
if (string.IsNullOrEmpty(subject))
|
||||
return string.Empty;
|
||||
|
||||
return TransformTokenized(subject.Split('.'));
|
||||
}
|
||||
|
||||
private string TransformTokenized(string[] tokens)
|
||||
{
|
||||
if (_ops.Length == 0)
|
||||
@@ -174,6 +289,10 @@ public sealed partial class SubjectTransform
|
||||
sb.Append(ComputePartition(tokens, op));
|
||||
break;
|
||||
|
||||
case TransformType.Random:
|
||||
sb.Append(GetRandomPartition(op.IntArg));
|
||||
break;
|
||||
|
||||
case TransformType.Split:
|
||||
ApplySplit(sb, tokens, op);
|
||||
break;
|
||||
@@ -252,6 +371,14 @@ public sealed partial class SubjectTransform
|
||||
return (hash % (uint)numBuckets).ToString();
|
||||
}
|
||||
|
||||
private static int GetRandomPartition(int numBuckets)
|
||||
{
|
||||
if (numBuckets <= 0)
|
||||
return 0;
|
||||
|
||||
return Random.Shared.Next(numBuckets);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// FNV-1a 32-bit hash. Offset basis: 2166136261, prime: 16777619.
|
||||
/// </summary>
|
||||
@@ -554,6 +681,19 @@ public sealed partial class SubjectTransform
|
||||
return new ParsedToken(TransformType.Partition, indexes, buckets, string.Empty);
|
||||
}
|
||||
|
||||
// random(numBuckets)
|
||||
args = GetFunctionArgs(RandomRegex(), token);
|
||||
if (args != null)
|
||||
{
|
||||
if (args.Length != 1)
|
||||
return null;
|
||||
|
||||
if (!TryParseInt32(args[0].Trim(), out int numBuckets))
|
||||
return null;
|
||||
|
||||
return new ParsedToken(TransformType.Random, [], numBuckets, string.Empty);
|
||||
}
|
||||
|
||||
// splitFromLeft(token, position)
|
||||
args = GetFunctionArgs(SplitFromLeftRegex(), token);
|
||||
if (args != null)
|
||||
@@ -623,6 +763,46 @@ public sealed partial class SubjectTransform
|
||||
return new ParsedToken(type, [idx], intArg, string.Empty);
|
||||
}
|
||||
|
||||
private static bool UsesAllSourceWildcards(string source, string destination)
|
||||
{
|
||||
var (srcValid, _, srcPwcCount, _) = SubjectInfo(source);
|
||||
if (!srcValid || srcPwcCount == 0)
|
||||
return true;
|
||||
|
||||
var (_, destTokens, _, _) = SubjectInfo(destination);
|
||||
var used = new HashSet<int>();
|
||||
|
||||
foreach (var token in destTokens)
|
||||
{
|
||||
var parsed = ParseDestToken(token);
|
||||
if (parsed == null)
|
||||
return false;
|
||||
|
||||
foreach (var wildcardIndex in parsed.WildcardIndexes)
|
||||
{
|
||||
if (wildcardIndex >= 1 && wildcardIndex <= srcPwcCount)
|
||||
used.Add(wildcardIndex);
|
||||
}
|
||||
}
|
||||
|
||||
for (var i = 1; i <= srcPwcCount; i++)
|
||||
{
|
||||
if (!used.Contains(i))
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool TryParseWildcardToken(string token, out int wildcardIndex)
|
||||
{
|
||||
wildcardIndex = 0;
|
||||
if (token.Length < 2 || token[0] != '$')
|
||||
return false;
|
||||
|
||||
return int.TryParse(token.AsSpan(1), out wildcardIndex) && wildcardIndex > 0;
|
||||
}
|
||||
|
||||
private static bool TryParseInt32(string s, out int result)
|
||||
{
|
||||
// Parse as long first to detect overflow
|
||||
@@ -655,6 +835,9 @@ public sealed partial class SubjectTransform
|
||||
[GeneratedRegex(@"\{\{\s*[pP]artition\s*\((.*)\)\s*\}\}")]
|
||||
private static partial Regex PartitionRegex();
|
||||
|
||||
[GeneratedRegex(@"\{\{\s*[rR]andom\s*\((.*)\)\s*\}\}")]
|
||||
private static partial Regex RandomRegex();
|
||||
|
||||
[GeneratedRegex(@"\{\{\s*[sS]plit[fF]rom[lL]eft\s*\((.*)\)\s*\}\}")]
|
||||
private static partial Regex SplitFromLeftRegex();
|
||||
|
||||
@@ -684,6 +867,7 @@ public sealed partial class SubjectTransform
|
||||
None,
|
||||
Wildcard,
|
||||
Partition,
|
||||
Random,
|
||||
Split,
|
||||
SplitFromLeft,
|
||||
SplitFromRight,
|
||||
|
||||
Reference in New Issue
Block a user