feat: complete final jetstream parity transport and runtime baselines
This commit is contained in:
29
tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs
Normal file
29
tests/NATS.Server.Tests/ClientKindProtocolRoutingTests.cs
Normal file
@@ -0,0 +1,29 @@
|
||||
using NATS.Server.Protocol;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class ClientKindProtocolRoutingTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(ClientKind.Client, "RS+", false)]
|
||||
[InlineData(ClientKind.Router, "RS+", true)]
|
||||
[InlineData(ClientKind.Client, "RS-", false)]
|
||||
[InlineData(ClientKind.Router, "RS-", true)]
|
||||
[InlineData(ClientKind.Client, "RMSG", false)]
|
||||
[InlineData(ClientKind.Router, "RMSG", true)]
|
||||
[InlineData(ClientKind.Client, "A+", false)]
|
||||
[InlineData(ClientKind.Gateway, "A+", true)]
|
||||
[InlineData(ClientKind.Client, "A-", false)]
|
||||
[InlineData(ClientKind.Gateway, "A-", true)]
|
||||
[InlineData(ClientKind.Client, "LS+", false)]
|
||||
[InlineData(ClientKind.Leaf, "LS+", true)]
|
||||
[InlineData(ClientKind.Client, "LS-", false)]
|
||||
[InlineData(ClientKind.Leaf, "LS-", true)]
|
||||
[InlineData(ClientKind.Client, "LMSG", false)]
|
||||
[InlineData(ClientKind.Leaf, "LMSG", true)]
|
||||
public void Client_kind_protocol_matrix_enforces_inter_server_commands(ClientKind kind, string op, bool expected)
|
||||
{
|
||||
var matrix = new ClientCommandMatrix();
|
||||
matrix.IsAllowed(kind, op).ShouldBe(expected);
|
||||
}
|
||||
}
|
||||
153
tests/NATS.Server.Tests/GatewayProtocolTests.cs
Normal file
153
tests/NATS.Server.Tests/GatewayProtocolTests.cs
Normal file
@@ -0,0 +1,153 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class GatewayProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Gateway_link_establishes_and_forwards_interested_message()
|
||||
{
|
||||
await using var fx = await GatewayFixture.StartTwoClustersAsync();
|
||||
await fx.SubscribeRemoteClusterAsync("g.>");
|
||||
await fx.PublishLocalClusterAsync("g.test", "hello");
|
||||
(await fx.ReadRemoteClusterMessageAsync()).ShouldContain("hello");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class GatewayFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _local;
|
||||
private readonly NatsServer _remote;
|
||||
private readonly CancellationTokenSource _localCts;
|
||||
private readonly CancellationTokenSource _remoteCts;
|
||||
private Socket? _remoteSubscriber;
|
||||
private Socket? _localPublisher;
|
||||
|
||||
private GatewayFixture(NatsServer local, NatsServer remote, CancellationTokenSource localCts, CancellationTokenSource remoteCts)
|
||||
{
|
||||
_local = local;
|
||||
_remote = remote;
|
||||
_localCts = localCts;
|
||||
_remoteCts = remoteCts;
|
||||
}
|
||||
|
||||
public static async Task<GatewayFixture> StartTwoClustersAsync()
|
||||
{
|
||||
var localOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Name = "LOCAL",
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var local = new NatsServer(localOptions, NullLoggerFactory.Instance);
|
||||
var localCts = new CancellationTokenSource();
|
||||
_ = local.StartAsync(localCts.Token);
|
||||
await local.WaitForReadyAsync();
|
||||
|
||||
var remoteOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Name = "REMOTE",
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Remotes = [local.GatewayListen!],
|
||||
},
|
||||
};
|
||||
|
||||
var remote = new NatsServer(remoteOptions, NullLoggerFactory.Instance);
|
||||
var remoteCts = new CancellationTokenSource();
|
||||
_ = remote.StartAsync(remoteCts.Token);
|
||||
await remote.WaitForReadyAsync();
|
||||
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested && (local.Stats.Gateways == 0 || remote.Stats.Gateways == 0))
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
|
||||
return new GatewayFixture(local, remote, localCts, remoteCts);
|
||||
}
|
||||
|
||||
public async Task SubscribeRemoteClusterAsync(string subject)
|
||||
{
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _remote.Port);
|
||||
_remoteSubscriber = sock;
|
||||
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task PublishLocalClusterAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _localPublisher;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _local.Port);
|
||||
_localPublisher = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public Task<string> ReadRemoteClusterMessageAsync()
|
||||
{
|
||||
if (_remoteSubscriber == null)
|
||||
throw new InvalidOperationException("Remote subscriber was not initialized.");
|
||||
|
||||
return ReadUntilAsync(_remoteSubscriber, "MSG ");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_remoteSubscriber?.Dispose();
|
||||
_localPublisher?.Dispose();
|
||||
await _localCts.CancelAsync();
|
||||
await _remoteCts.CancelAsync();
|
||||
_local.Dispose();
|
||||
_remote.Dispose();
|
||||
_localCts.Dispose();
|
||||
_remoteCts.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<string> ReadLineAsync(Socket sock)
|
||||
{
|
||||
var buf = new byte[4096];
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
|
||||
return Encoding.ASCII.GetString(buf, 0, n);
|
||||
}
|
||||
|
||||
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var buf = new byte[4096];
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
|
||||
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
|
||||
{
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
|
||||
if (n == 0)
|
||||
break;
|
||||
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
17
tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs
Normal file
17
tests/NATS.Server.Tests/JetStreamAccountControlApiTests.cs
Normal file
@@ -0,0 +1,17 @@
|
||||
using NATS.Server.JetStream;
|
||||
using NATS.Server.JetStream.Api;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamAccountControlApiTests
|
||||
{
|
||||
[Fact]
|
||||
public void Account_and_server_control_subjects_are_routable()
|
||||
{
|
||||
var router = new JetStreamApiRouter(new StreamManager(), new ConsumerManager());
|
||||
router.Route("$JS.API.SERVER.REMOVE", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.PURGE.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
router.Route("$JS.API.ACCOUNT.STREAM.MOVE.CANCEL.ACC", "{}"u8).Error.ShouldBeNull();
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,20 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartWithStreamConfigAsync(StreamConfig config)
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = fixture._streamManager.CreateOrUpdate(config);
|
||||
return Task.FromResult(fixture);
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithStreamJsonAsync(string json)
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = await fixture.RequestLocalAsync("$JS.API.STREAM.CREATE.S", json);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithPullConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
|
||||
@@ -82,6 +96,47 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithMultiFilterConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", ">");
|
||||
_ = await fixture.CreateConsumerAsync("ORDERS", "CF", null, filterSubjects: ["orders.*"]);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static async Task<JetStreamApiFixture> StartWithReplayOriginalConsumerAsync()
|
||||
{
|
||||
var fixture = await StartWithStreamAsync("ORDERS", "orders.*");
|
||||
_ = await fixture.PublishAndGetAckAsync("orders.created", "1");
|
||||
_ = await fixture.CreateConsumerAsync("ORDERS", "RO", "orders.*", replayPolicy: ReplayPolicy.Original, ackPolicy: AckPolicy.Explicit);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartWithMultipleSourcesAsync()
|
||||
{
|
||||
var fixture = new JetStreamApiFixture();
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "SRC1",
|
||||
Subjects = ["a.>"],
|
||||
});
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "SRC2",
|
||||
Subjects = ["b.>"],
|
||||
});
|
||||
_ = fixture._streamManager.CreateOrUpdate(new StreamConfig
|
||||
{
|
||||
Name = "AGG",
|
||||
Subjects = ["agg.>"],
|
||||
Sources =
|
||||
[
|
||||
new StreamSourceConfig { Name = "SRC1" },
|
||||
new StreamSourceConfig { Name = "SRC2" },
|
||||
],
|
||||
});
|
||||
return Task.FromResult(fixture);
|
||||
}
|
||||
|
||||
public static Task<JetStreamApiFixture> StartJwtLimitedAccountAsync(int maxStreams)
|
||||
{
|
||||
var account = new Account("JWT-LIMITED")
|
||||
@@ -148,9 +203,45 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
return _streamManager.GetStateAsync(streamName, default).AsTask();
|
||||
}
|
||||
|
||||
public Task<JetStreamApiResponse> CreateConsumerAsync(string stream, string durableName, string filterSubject, bool push = false, int heartbeatMs = 0, AckPolicy ackPolicy = AckPolicy.None, int ackWaitMs = 30_000)
|
||||
public Task<string> GetStreamBackendTypeAsync(string streamName)
|
||||
{
|
||||
var payload = $@"{{""durable_name"":""{durableName}"",""filter_subject"":""{filterSubject}"",""push"":{push.ToString().ToLowerInvariant()},""heartbeat_ms"":{heartbeatMs},""ack_policy"":""{ackPolicy.ToString().ToLowerInvariant()}"",""ack_wait_ms"":{ackWaitMs}}}";
|
||||
return Task.FromResult(_streamManager.GetStoreBackendType(streamName));
|
||||
}
|
||||
|
||||
public Task<JetStreamApiResponse> CreateConsumerAsync(
|
||||
string stream,
|
||||
string durableName,
|
||||
string? filterSubject,
|
||||
bool push = false,
|
||||
int heartbeatMs = 0,
|
||||
AckPolicy ackPolicy = AckPolicy.None,
|
||||
int ackWaitMs = 30_000,
|
||||
int maxAckPending = 0,
|
||||
IReadOnlyList<string>? filterSubjects = null,
|
||||
ReplayPolicy replayPolicy = ReplayPolicy.Instant,
|
||||
DeliverPolicy deliverPolicy = DeliverPolicy.All,
|
||||
bool ephemeral = false)
|
||||
{
|
||||
var payloadObj = new
|
||||
{
|
||||
durable_name = durableName,
|
||||
filter_subject = filterSubject,
|
||||
filter_subjects = filterSubjects,
|
||||
push,
|
||||
heartbeat_ms = heartbeatMs,
|
||||
ack_policy = ackPolicy.ToString().ToLowerInvariant(),
|
||||
ack_wait_ms = ackWaitMs,
|
||||
max_ack_pending = maxAckPending,
|
||||
replay_policy = replayPolicy == ReplayPolicy.Original ? "original" : "instant",
|
||||
deliver_policy = deliverPolicy switch
|
||||
{
|
||||
DeliverPolicy.Last => "last",
|
||||
DeliverPolicy.New => "new",
|
||||
_ => "all",
|
||||
},
|
||||
ephemeral,
|
||||
};
|
||||
var payload = JsonSerializer.Serialize(payloadObj);
|
||||
return RequestLocalAsync($"$JS.API.CONSUMER.CREATE.{stream}.{durableName}", payload);
|
||||
}
|
||||
|
||||
@@ -206,6 +297,12 @@ internal sealed class JetStreamApiFixture : IAsyncDisposable
|
||||
_ = await PublishAndGetAckAsync(subject, payload);
|
||||
}
|
||||
|
||||
public Task PublishToSourceAsync(string sourceStream, string subject, string payload)
|
||||
{
|
||||
_ = sourceStream;
|
||||
return PublishAndGetAckAsync(subject, payload);
|
||||
}
|
||||
|
||||
public Task AckAllAsync(string stream, string durableName, ulong sequence)
|
||||
{
|
||||
_consumerManager.AckAll(stream, durableName, sequence);
|
||||
|
||||
88
tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs
Normal file
88
tests/NATS.Server.Tests/JetStreamApiGapInventoryTests.cs
Normal file
@@ -0,0 +1,88 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamApiGapInventoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void Parity_map_has_no_unclassified_go_js_api_subjects()
|
||||
{
|
||||
var gap = JetStreamApiGapInventory.Load();
|
||||
gap.UnclassifiedSubjects.ShouldBeEmpty();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class JetStreamApiGapInventory
|
||||
{
|
||||
public IReadOnlyList<string> UnclassifiedSubjects { get; }
|
||||
|
||||
private JetStreamApiGapInventory(IReadOnlyList<string> unclassifiedSubjects)
|
||||
{
|
||||
UnclassifiedSubjects = unclassifiedSubjects;
|
||||
}
|
||||
|
||||
public static JetStreamApiGapInventory Load()
|
||||
{
|
||||
var goSubjects = LoadGoSubjects();
|
||||
var mappedSubjects = LoadMappedSubjects();
|
||||
|
||||
var unclassified = goSubjects
|
||||
.Where(s => !mappedSubjects.Contains(s))
|
||||
.OrderBy(s => s, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return new JetStreamApiGapInventory(unclassified);
|
||||
}
|
||||
|
||||
private static HashSet<string> LoadGoSubjects()
|
||||
{
|
||||
var script = Path.Combine(AppContext.BaseDirectory, "../../../../../scripts/jetstream/extract-go-js-api.sh");
|
||||
script = Path.GetFullPath(script);
|
||||
if (!File.Exists(script))
|
||||
throw new FileNotFoundException($"missing script: {script}");
|
||||
|
||||
var psi = new ProcessStartInfo
|
||||
{
|
||||
FileName = "bash",
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
};
|
||||
psi.ArgumentList.Add(script);
|
||||
|
||||
using var process = Process.Start(psi) ?? throw new InvalidOperationException("failed to start inventory script");
|
||||
var output = process.StandardOutput.ReadToEnd();
|
||||
var errors = process.StandardError.ReadToEnd();
|
||||
process.WaitForExit();
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
throw new InvalidOperationException($"inventory script failed: {errors}");
|
||||
|
||||
return output
|
||||
.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Where(x => x.StartsWith("$JS.API.", StringComparison.Ordinal))
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static HashSet<string> LoadMappedSubjects()
|
||||
{
|
||||
var mapPath = Path.Combine(AppContext.BaseDirectory, "../../../../../docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
|
||||
mapPath = Path.GetFullPath(mapPath);
|
||||
if (!File.Exists(mapPath))
|
||||
throw new FileNotFoundException($"missing parity map: {mapPath}");
|
||||
|
||||
var subjectRegex = new Regex(@"^\|\s*(\$JS\.API[^\|]+?)\s*\|", RegexOptions.Compiled);
|
||||
var subjects = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var line in File.ReadLines(mapPath))
|
||||
{
|
||||
var match = subjectRegex.Match(line);
|
||||
if (!match.Success)
|
||||
continue;
|
||||
|
||||
subjects.Add(match.Groups[1].Value.Trim());
|
||||
}
|
||||
|
||||
return subjects;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamClusterControlExtendedApiTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Peer_remove_and_consumer_stepdown_subjects_return_success_shape()
|
||||
{
|
||||
await using var fx = await JetStreamClusterFixture.StartAsync(nodes: 3);
|
||||
(await fx.RequestAsync("$JS.API.STREAM.PEER.REMOVE.ORDERS", "{\"peer\":\"n2\"}")).Success.ShouldBeTrue();
|
||||
(await fx.RequestAsync("$JS.API.CONSUMER.LEADER.STEPDOWN.ORDERS.DUR", "{}")).Success.ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
16
tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs
Normal file
16
tests/NATS.Server.Tests/JetStreamConsumerSemanticsTests.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamConsumerSemanticsTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Consumer_with_filter_subjects_only_receives_matching_messages()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultiFilterConsumerAsync();
|
||||
await fx.PublishAndGetAckAsync("orders.created", "1");
|
||||
await fx.PublishAndGetAckAsync("payments.settled", "2");
|
||||
|
||||
var batch = await fx.FetchAsync("ORDERS", "CF", 10);
|
||||
batch.Messages.ShouldNotBeEmpty();
|
||||
batch.Messages.All(m => m.Subject.StartsWith("orders.", StringComparison.Ordinal)).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
15
tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs
Normal file
15
tests/NATS.Server.Tests/JetStreamFlowReplayBackoffTests.cs
Normal file
@@ -0,0 +1,15 @@
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamFlowReplayBackoffTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Replay_original_respects_message_timestamps_with_backoff_redelivery()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithReplayOriginalConsumerAsync();
|
||||
var sw = Stopwatch.StartNew();
|
||||
_ = await fx.FetchAsync("ORDERS", "RO", 1);
|
||||
sw.ElapsedMilliseconds.ShouldBeGreaterThanOrEqualTo(50);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamMirrorSourceAdvancedTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Stream_with_multiple_sources_aggregates_messages_in_order()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithMultipleSourcesAsync();
|
||||
await fx.PublishToSourceAsync("SRC1", "a.1", "1");
|
||||
await fx.PublishToSourceAsync("SRC2", "b.1", "2");
|
||||
|
||||
(await fx.GetStreamStateAsync("AGG")).Messages.ShouldBe((ulong)2);
|
||||
}
|
||||
}
|
||||
11
tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs
Normal file
11
tests/NATS.Server.Tests/JetStreamStorageSelectionTests.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamStorageSelectionTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Stream_with_storage_file_uses_filestore_backend()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamJsonAsync("{\"name\":\"S\",\"subjects\":[\"s.*\"],\"storage\":\"file\"}");
|
||||
(await fx.GetStreamBackendTypeAsync("S")).ShouldBe("file");
|
||||
}
|
||||
}
|
||||
21
tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs
Normal file
21
tests/NATS.Server.Tests/JetStreamStreamPolicyRuntimeTests.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
using NATS.Server.JetStream.Models;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class JetStreamStreamPolicyRuntimeTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Discard_new_rejects_publish_when_max_bytes_exceeded()
|
||||
{
|
||||
await using var fx = await JetStreamApiFixture.StartWithStreamConfigAsync(new StreamConfig
|
||||
{
|
||||
Name = "S",
|
||||
Subjects = ["s.*"],
|
||||
MaxBytes = 2,
|
||||
Discard = DiscardPolicy.New,
|
||||
});
|
||||
|
||||
(await fx.PublishAndGetAckAsync("s.a", "12")).ErrorCode.ShouldBeNull();
|
||||
(await fx.PublishAndGetAckAsync("s.a", "34")).ErrorCode.ShouldNotBeNull();
|
||||
}
|
||||
}
|
||||
151
tests/NATS.Server.Tests/LeafProtocolTests.cs
Normal file
151
tests/NATS.Server.Tests/LeafProtocolTests.cs
Normal file
@@ -0,0 +1,151 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class LeafProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Leaf_link_propagates_subscription_and_message_flow()
|
||||
{
|
||||
await using var fx = await LeafFixture.StartHubSpokeAsync();
|
||||
await fx.SubscribeSpokeAsync("leaf.>");
|
||||
await fx.PublishHubAsync("leaf.msg", "x");
|
||||
(await fx.ReadSpokeMessageAsync()).ShouldContain("x");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class LeafFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _hub;
|
||||
private readonly NatsServer _spoke;
|
||||
private readonly CancellationTokenSource _hubCts;
|
||||
private readonly CancellationTokenSource _spokeCts;
|
||||
private Socket? _spokeSubscriber;
|
||||
private Socket? _hubPublisher;
|
||||
|
||||
private LeafFixture(NatsServer hub, NatsServer spoke, CancellationTokenSource hubCts, CancellationTokenSource spokeCts)
|
||||
{
|
||||
_hub = hub;
|
||||
_spoke = spoke;
|
||||
_hubCts = hubCts;
|
||||
_spokeCts = spokeCts;
|
||||
}
|
||||
|
||||
public static async Task<LeafFixture> StartHubSpokeAsync()
|
||||
{
|
||||
var hubOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var hub = new NatsServer(hubOptions, NullLoggerFactory.Instance);
|
||||
var hubCts = new CancellationTokenSource();
|
||||
_ = hub.StartAsync(hubCts.Token);
|
||||
await hub.WaitForReadyAsync();
|
||||
|
||||
var spokeOptions = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Remotes = [hub.LeafListen!],
|
||||
},
|
||||
};
|
||||
|
||||
var spoke = new NatsServer(spokeOptions, NullLoggerFactory.Instance);
|
||||
var spokeCts = new CancellationTokenSource();
|
||||
_ = spoke.StartAsync(spokeCts.Token);
|
||||
await spoke.WaitForReadyAsync();
|
||||
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested && (hub.Stats.Leafs == 0 || spoke.Stats.Leafs == 0))
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
|
||||
return new LeafFixture(hub, spoke, hubCts, spokeCts);
|
||||
}
|
||||
|
||||
public async Task SubscribeSpokeAsync(string subject)
|
||||
{
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _spoke.Port);
|
||||
_spokeSubscriber = sock;
|
||||
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"CONNECT {{}}\r\nSUB {subject} 1\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task PublishHubAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _hubPublisher;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _hub.Port);
|
||||
_hubPublisher = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public Task<string> ReadSpokeMessageAsync()
|
||||
{
|
||||
if (_spokeSubscriber == null)
|
||||
throw new InvalidOperationException("Spoke subscriber was not initialized.");
|
||||
|
||||
return ReadUntilAsync(_spokeSubscriber, "MSG ");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_spokeSubscriber?.Dispose();
|
||||
_hubPublisher?.Dispose();
|
||||
await _hubCts.CancelAsync();
|
||||
await _spokeCts.CancelAsync();
|
||||
_hub.Dispose();
|
||||
_spoke.Dispose();
|
||||
_hubCts.Dispose();
|
||||
_spokeCts.Dispose();
|
||||
}
|
||||
|
||||
private static async Task<string> ReadLineAsync(Socket sock)
|
||||
{
|
||||
var buf = new byte[4096];
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None);
|
||||
return Encoding.ASCII.GetString(buf, 0, n);
|
||||
}
|
||||
|
||||
private static async Task<string> ReadUntilAsync(Socket sock, string expected)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var buf = new byte[4096];
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
|
||||
while (!sb.ToString().Contains(expected, StringComparison.Ordinal))
|
||||
{
|
||||
var n = await sock.ReceiveAsync(buf, SocketFlags.None, cts.Token);
|
||||
if (n == 0)
|
||||
break;
|
||||
sb.Append(Encoding.ASCII.GetString(buf, 0, n));
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
}
|
||||
105
tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs
Normal file
105
tests/NATS.Server.Tests/MonitorClusterEndpointTests.cs
Normal file
@@ -0,0 +1,105 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NATS.Server.Configuration;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class MonitorClusterEndpointTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Routez_gatewayz_leafz_accountz_return_non_stub_runtime_data()
|
||||
{
|
||||
await using var fx = await MonitorFixture.StartClusterEnabledAsync();
|
||||
(await fx.GetJsonAsync("/routez")).ShouldContain("routes");
|
||||
(await fx.GetJsonAsync("/gatewayz")).ShouldContain("gateways");
|
||||
(await fx.GetJsonAsync("/leafz")).ShouldContain("leafs");
|
||||
(await fx.GetJsonAsync("/accountz")).ShouldContain("accounts");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MonitorFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly NatsServer _server;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly HttpClient _http;
|
||||
private readonly int _monitorPort;
|
||||
|
||||
private MonitorFixture(NatsServer server, CancellationTokenSource cts, HttpClient http, int monitorPort)
|
||||
{
|
||||
_server = server;
|
||||
_cts = cts;
|
||||
_http = http;
|
||||
_monitorPort = monitorPort;
|
||||
}
|
||||
|
||||
public static async Task<MonitorFixture> StartClusterEnabledAsync()
|
||||
{
|
||||
var monitorPort = GetFreePort();
|
||||
var options = new NatsOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
MonitorPort = monitorPort,
|
||||
Cluster = new ClusterOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
Gateway = new GatewayOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
Name = "M",
|
||||
},
|
||||
LeafNode = new LeafNodeOptions
|
||||
{
|
||||
Host = "127.0.0.1",
|
||||
Port = 0,
|
||||
},
|
||||
};
|
||||
|
||||
var server = new NatsServer(options, NullLoggerFactory.Instance);
|
||||
var cts = new CancellationTokenSource();
|
||||
_ = server.StartAsync(cts.Token);
|
||||
await server.WaitForReadyAsync();
|
||||
|
||||
var http = new HttpClient();
|
||||
for (var i = 0; i < 50; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await http.GetAsync($"http://127.0.0.1:{monitorPort}/healthz");
|
||||
if (response.IsSuccessStatusCode)
|
||||
break;
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
await Task.Delay(50);
|
||||
}
|
||||
|
||||
return new MonitorFixture(server, cts, http, monitorPort);
|
||||
}
|
||||
|
||||
public Task<string> GetJsonAsync(string path)
|
||||
{
|
||||
return _http.GetStringAsync($"http://127.0.0.1:{_monitorPort}{path}");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_http.Dispose();
|
||||
await _cts.CancelAsync();
|
||||
_server.Dispose();
|
||||
_cts.Dispose();
|
||||
}
|
||||
|
||||
private static int GetFreePort()
|
||||
{
|
||||
using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
|
||||
return ((IPEndPoint)socket.LocalEndPoint!).Port;
|
||||
}
|
||||
}
|
||||
89
tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs
Normal file
89
tests/NATS.Server.Tests/RaftTransportPersistenceTests.cs
Normal file
@@ -0,0 +1,89 @@
|
||||
using NATS.Server.Raft;
|
||||
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RaftTransportPersistenceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Raft_node_recovers_log_and_term_after_restart()
|
||||
{
|
||||
await using var fx = await RaftFixture.StartPersistentClusterAsync();
|
||||
var idx = await fx.Leader.ProposeAsync("cmd", default);
|
||||
await fx.RestartNodeAsync("n2");
|
||||
(await fx.ReadNodeAppliedIndexAsync("n2")).ShouldBeGreaterThanOrEqualTo(idx);
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class RaftFixture : IAsyncDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
private readonly InMemoryRaftTransport _transport;
|
||||
private readonly Dictionary<string, RaftNode> _nodes;
|
||||
|
||||
private RaftFixture(string root, InMemoryRaftTransport transport, Dictionary<string, RaftNode> nodes)
|
||||
{
|
||||
_root = root;
|
||||
_transport = transport;
|
||||
_nodes = nodes;
|
||||
}
|
||||
|
||||
public RaftNode Leader => _nodes["n1"];
|
||||
|
||||
public static Task<RaftFixture> StartPersistentClusterAsync()
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), $"nats-raft-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
var transport = new InMemoryRaftTransport();
|
||||
var nodes = new Dictionary<string, RaftNode>(StringComparer.Ordinal);
|
||||
foreach (var id in new[] { "n1", "n2", "n3" })
|
||||
{
|
||||
var node = new RaftNode(id, transport, Path.Combine(root, id));
|
||||
transport.Register(node);
|
||||
nodes[id] = node;
|
||||
}
|
||||
|
||||
var all = nodes.Values.ToArray();
|
||||
foreach (var node in all)
|
||||
node.ConfigureCluster(all);
|
||||
|
||||
var leader = nodes["n1"];
|
||||
leader.StartElection(all.Length);
|
||||
leader.ReceiveVote(nodes["n2"].GrantVote(leader.Term), all.Length);
|
||||
leader.ReceiveVote(nodes["n3"].GrantVote(leader.Term), all.Length);
|
||||
|
||||
return Task.FromResult(new RaftFixture(root, transport, nodes));
|
||||
}
|
||||
|
||||
public async Task RestartNodeAsync(string id)
|
||||
{
|
||||
var nodePath = Path.Combine(_root, id);
|
||||
var replacement = new RaftNode(id, _transport, nodePath);
|
||||
await replacement.LoadPersistedStateAsync(default);
|
||||
_transport.Register(replacement);
|
||||
_nodes[id] = replacement;
|
||||
|
||||
var all = _nodes.Values.ToArray();
|
||||
foreach (var node in all)
|
||||
node.ConfigureCluster(all);
|
||||
}
|
||||
|
||||
public Task<long> ReadNodeAppliedIndexAsync(string id)
|
||||
{
|
||||
return Task.FromResult(_nodes[id].AppliedIndex);
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
11
tests/NATS.Server.Tests/RoutePoolTests.cs
Normal file
11
tests/NATS.Server.Tests/RoutePoolTests.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RoutePoolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Route_manager_establishes_default_pool_of_three_links_per_peer()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
(await fx.ServerARouteLinkCountToServerBAsync()).ShouldBe(3);
|
||||
}
|
||||
}
|
||||
14
tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs
Normal file
14
tests/NATS.Server.Tests/RouteRmsgForwardingTests.cs
Normal file
@@ -0,0 +1,14 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RouteRmsgForwardingTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Publish_on_serverA_reaches_remote_subscriber_on_serverB_via_RMSG()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
await fx.SubscribeOnServerBAsync("foo.>");
|
||||
|
||||
await fx.PublishFromServerAAsync("foo.bar", "payload");
|
||||
(await fx.ReadServerBMessageAsync()).ShouldContain("payload");
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,8 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
private readonly CancellationTokenSource _ctsA;
|
||||
private readonly CancellationTokenSource _ctsB;
|
||||
private Socket? _subscriberOnB;
|
||||
private Socket? _publisherOnA;
|
||||
private Socket? _manualRouteToA;
|
||||
|
||||
private RouteFixture(NatsServer serverA, NatsServer serverB, CancellationTokenSource ctsA, CancellationTokenSource ctsB)
|
||||
{
|
||||
@@ -91,22 +93,82 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task<bool> ServerAHasRemoteInterestAsync(string subject)
|
||||
public async Task SendRouteSubFrameAsync(string subject)
|
||||
{
|
||||
var (host, port) = ParseHostPort(_serverA.ClusterListen!);
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Parse(host), port);
|
||||
_manualRouteToA = sock;
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("ROUTE test-remote\r\n"));
|
||||
_ = await ReadLineAsync(sock); // ROUTE <id>
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"RS+ {subject}\r\n"));
|
||||
}
|
||||
|
||||
public async Task SendRouteUnsubFrameAsync(string subject)
|
||||
{
|
||||
if (_manualRouteToA == null)
|
||||
throw new InvalidOperationException("Route frame socket not established.");
|
||||
|
||||
await _manualRouteToA.SendAsync(Encoding.ASCII.GetBytes($"RS- {subject}\r\n"));
|
||||
}
|
||||
|
||||
public async Task PublishFromServerAAsync(string subject, string payload)
|
||||
{
|
||||
var sock = _publisherOnA;
|
||||
if (sock == null)
|
||||
{
|
||||
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
|
||||
await sock.ConnectAsync(IPAddress.Loopback, _serverA.Port);
|
||||
_publisherOnA = sock;
|
||||
_ = await ReadLineAsync(sock); // INFO
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes("CONNECT {}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
await sock.SendAsync(Encoding.ASCII.GetBytes($"PUB {subject} {payload.Length}\r\n{payload}\r\nPING\r\n"));
|
||||
await ReadUntilAsync(sock, "PONG");
|
||||
}
|
||||
|
||||
public async Task<string> ReadServerBMessageAsync()
|
||||
{
|
||||
if (_subscriberOnB == null)
|
||||
throw new InvalidOperationException("No subscriber socket on server B.");
|
||||
|
||||
return await ReadUntilAsync(_subscriberOnB, "MSG ");
|
||||
}
|
||||
|
||||
public async Task<bool> ServerAHasRemoteInterestAsync(string subject, bool expected = true)
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested)
|
||||
{
|
||||
if (_serverA.HasRemoteInterest(subject))
|
||||
return true;
|
||||
if (_serverA.HasRemoteInterest(subject) == expected)
|
||||
return expected;
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
return false;
|
||||
return !expected;
|
||||
}
|
||||
|
||||
public async Task<int> ServerARouteLinkCountToServerBAsync()
|
||||
{
|
||||
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
|
||||
while (!timeout.IsCancellationRequested)
|
||||
{
|
||||
if (_serverA.Stats.Routes >= 3)
|
||||
return (int)_serverA.Stats.Routes;
|
||||
await Task.Delay(50, timeout.Token).ContinueWith(_ => { }, TaskScheduler.Default);
|
||||
}
|
||||
|
||||
return (int)_serverA.Stats.Routes;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
_subscriberOnB?.Dispose();
|
||||
_publisherOnA?.Dispose();
|
||||
_manualRouteToA?.Dispose();
|
||||
await _ctsA.CancelAsync();
|
||||
await _ctsB.CancelAsync();
|
||||
_serverA.Dispose();
|
||||
@@ -138,4 +200,10 @@ internal sealed class RouteFixture : IAsyncDisposable
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static (string Host, int Port) ParseHostPort(string endpoint)
|
||||
{
|
||||
var parts = endpoint.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
return (parts[0], int.Parse(parts[1]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace NATS.Server.Tests;
|
||||
|
||||
public class RouteWireSubscriptionProtocolTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RSplus_RSminus_frames_propagate_remote_interest_over_socket()
|
||||
{
|
||||
await using var fx = await RouteFixture.StartTwoNodeClusterAsync();
|
||||
|
||||
await fx.SendRouteSubFrameAsync("foo.*");
|
||||
(await fx.ServerAHasRemoteInterestAsync("foo.bar")).ShouldBeTrue();
|
||||
|
||||
await fx.SendRouteUnsubFrameAsync("foo.*");
|
||||
(await fx.ServerAHasRemoteInterestAsync("foo.bar", expected: false)).ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user