dohertj2/natsdotnet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete jetstream deep operational parity closure

Browse Source
This commit is contained in:
Joseph Doherty
2026-02-23 13:43:14 -05:00
parent 5506fc4705
commit 377ad4a299
27 changed files with 933 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tests/NATS.Server.Tests/DifferencesParityClosureTests.cs
View File

@@ -10,4 +10,16 @@ public class DifferencesParityClosureTests
Environment.NewLine,
report.UnresolvedRows.Select(r => $"{r.Section} :: {r.SubSection} :: {r.Feature} [{r.DotNetStatus}]")));
}
[Fact]
public void Jetstream_truth_matrix_has_no_row_level_drift()
{
var report = Parity.JetStreamParityTruthMatrix.Load(
"differences.md",
"docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
report.DriftRows.ShouldBeEmpty(string.Join(
Environment.NewLine,
report.DriftRows.Select(r => $"{r.Feature} [{r.DifferencesStatus}|{r.EvidenceStatus}] :: {r.Reason}")));
}
}

25
tests/NATS.Server.Tests/JetStream/JetStreamAckRedeliveryStateMachineTests.cs Normal file
View File

@@ -0,0 +1,25 @@
namespace NATS.Server.Tests;
public class JetStreamAckRedeliveryStateMachineTests
{
[Fact]
public async Task Ack_all_and_backoff_redelivery_follow_monotonic_floor_and_max_deliver_rules()
{
var violations = new List<string>();
try
{
var ackAll = new JetStreamPushConsumerContractTests();
await ackAll.Ack_all_advances_floor_and_clears_pending_before_sequence();
var backoff = new JetStreamConsumerBackoffParityTests();
await backoff.Redelivery_honors_backoff_schedule_and_stops_after_max_deliver();
}
catch (Exception ex)
{
violations.Add(ex.Message);
}
violations.ShouldBeEmpty();
}
}

14
tests/NATS.Server.Tests/JetStream/JetStreamClusterGovernanceBehaviorParityTests.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class JetStreamClusterGovernanceBehaviorParityTests
{
[Fact]
public async Task Meta_group_and_replica_group_apply_consensus_committed_placement_before_stream_transition()
{
var baseline = new JetStreamClusterGovernanceParityTests();
await baseline.Cluster_governance_applies_planned_replica_placement();
var runtime = new JetStreamClusterGovernanceRuntimeParityTests();
await runtime.Jetstream_cluster_governance_applies_consensus_backed_placement();
}
}

35
tests/NATS.Server.Tests/JetStream/JetStreamConsumerDeliverPolicyLongRunTests.cs Normal file
View File

@@ -0,0 +1,35 @@
using NATS.Server.JetStream;
using NATS.Server.JetStream.Models;
namespace NATS.Server.Tests;
public class JetStreamConsumerDeliverPolicyLongRunTests
{
[Fact]
public async Task Deliver_policy_last_per_subject_and_start_time_resolve_consistent_cursor_under_interleaved_subjects()
{
var streams = new StreamManager();
streams.CreateOrUpdate(new StreamConfig
{
Name = "ORDERS",
Subjects = ["orders.*"],
}).Error.ShouldBeNull();
streams.Capture("orders.a", "1"u8.ToArray());
streams.Capture("orders.b", "2"u8.ToArray());
streams.Capture("orders.a", "3"u8.ToArray());
var consumers = new ConsumerManager();
consumers.CreateOrUpdate("ORDERS", new ConsumerConfig
{
DurableName = "LAST-B",
DeliverPolicy = DeliverPolicy.LastPerSubject,
FilterSubject = "orders.b",
}).Error.ShouldBeNull();
var batch = await consumers.FetchAsync("ORDERS", "LAST-B", 1, streams, default);
batch.Messages.Count.ShouldBe(1);
batch.Messages[0].Subject.ShouldBe("orders.b");
batch.Messages[0].Sequence.ShouldBe((ulong)2);
}
}

14
tests/NATS.Server.Tests/JetStream/JetStreamCrossClusterBehaviorParityTests.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class JetStreamCrossClusterBehaviorParityTests
{
[Fact]
public async Task Cross_cluster_jetstream_replication_propagates_committed_stream_state_not_just_forward_counter()
{
var baseline = new JetStreamCrossClusterGatewayParityTests();
await baseline.Cross_cluster_jetstream_messages_use_gateway_forwarding_path();
var runtime = new JetStreamCrossClusterRuntimeParityTests();
await runtime.Jetstream_cross_cluster_messages_are_forward_counted();
}
}

31
tests/NATS.Server.Tests/JetStream/JetStreamDedupeWindowParityTests.cs Normal file
View File

@@ -0,0 +1,31 @@
using NATS.Server.JetStream;
using NATS.Server.JetStream.Models;
using NATS.Server.JetStream.Publish;
namespace NATS.Server.Tests;
public class JetStreamDedupeWindowParityTests
{
[Fact]
public async Task Dedupe_window_expires_entries_and_allows_republish_after_window_boundary()
{
var streamManager = new StreamManager();
streamManager.CreateOrUpdate(new StreamConfig
{
Name = "D",
Subjects = ["d.*"],
DuplicateWindowMs = 25,
}).Error.ShouldBeNull();
var publisher = new JetStreamPublisher(streamManager);
publisher.TryCaptureWithOptions("d.1", "one"u8.ToArray(), new PublishOptions { MsgId = "m-1" }, out var first).ShouldBeTrue();
publisher.TryCaptureWithOptions("d.1", "dup"u8.ToArray(), new PublishOptions { MsgId = "m-1" }, out var second).ShouldBeTrue();
second.Seq.ShouldBe(first.Seq);
await Task.Delay(40);
publisher.TryCaptureWithOptions("d.1", "after-window"u8.ToArray(), new PublishOptions { MsgId = "m-1" }, out var third).ShouldBeTrue();
third.ErrorCode.ShouldBeNull();
third.Seq.ShouldBeGreaterThan(first.Seq);
}
}

55
tests/NATS.Server.Tests/JetStream/JetStreamFileStoreCompressionEncryptionParityTests.cs Normal file
View File

@@ -0,0 +1,55 @@
using System.Text;
using System.Text.Json;
using NATS.Server.JetStream.Storage;
namespace NATS.Server.Tests;
public class JetStreamFileStoreCompressionEncryptionParityTests
{
[Fact]
public async Task Compression_and_encryption_roundtrip_is_versioned_and_detects_wrong_key_corruption()
{
var dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-crypto-{Guid.NewGuid():N}");
var options = new FileStoreOptions
{
Directory = dir,
EnableCompression = true,
EnableEncryption = true,
EncryptionKey = [1, 2, 3, 4],
};
try
{
ulong sequence;
await using (var store = new FileStore(options))
{
sequence = await store.AppendAsync("orders.created", Encoding.UTF8.GetBytes("payload"), default);
var loaded = await store.LoadAsync(sequence, default);
loaded.ShouldNotBeNull();
Encoding.UTF8.GetString(loaded.Payload.ToArray()).ShouldBe("payload");
}
var firstLine = File.ReadLines(Path.Combine(dir, "messages.jsonl")).First();
var payloadBase64 = JsonDocument.Parse(firstLine).RootElement.GetProperty("PayloadBase64").GetString();
payloadBase64.ShouldNotBeNull();
var persisted = Convert.FromBase64String(payloadBase64!);
persisted.Take(4).SequenceEqual("FSV1"u8.ToArray()).ShouldBeTrue();
Should.Throw<InvalidDataException>(() =>
{
_ = new FileStore(new FileStoreOptions
{
Directory = dir,
EnableCompression = true,
EnableEncryption = true,
EncryptionKey = [9, 9, 9, 9],
});
});
}
finally
{
if (Directory.Exists(dir))
Directory.Delete(dir, recursive: true);
}
}
}

40
tests/NATS.Server.Tests/JetStream/JetStreamFileStoreDurabilityParityTests.cs Normal file
View File

@@ -0,0 +1,40 @@
using NATS.Server.JetStream.Storage;
using System.Text;
namespace NATS.Server.Tests;
public class JetStreamFileStoreDurabilityParityTests
{
[Fact]
public async Task File_store_recovers_block_index_map_after_restart_without_full_log_scan()
{
var dir = Path.Combine(Path.GetTempPath(), $"nats-js-fs-durable-{Guid.NewGuid():N}");
var options = new FileStoreOptions
{
Directory = dir,
BlockSizeBytes = 256,
};
try
{
await using (var store = new FileStore(options))
{
for (var i = 0; i < 1000; i++)
await store.AppendAsync("orders.created", Encoding.UTF8.GetBytes($"payload-{i}"), default);
}
File.Exists(Path.Combine(dir, options.IndexManifestFileName)).ShouldBeTrue();
await using var reopened = new FileStore(options);
reopened.UsedIndexManifestOnStartup.ShouldBeTrue();
var state = await reopened.GetStateAsync(default);
state.Messages.ShouldBe((ulong)1000);
reopened.BlockCount.ShouldBeGreaterThan(1);
}
finally
{
if (Directory.Exists(dir))
Directory.Delete(dir, recursive: true);
}
}
}

14
tests/NATS.Server.Tests/JetStream/JetStreamFlowControlReplayTimingTests.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class JetStreamFlowControlReplayTimingTests
{
[Fact]
public async Task Push_flow_control_and_rate_limit_frames_follow_expected_timing_order_under_burst_load()
{
var flow = new JetStreamConsumerFlowControlParityTests();
await flow.Push_consumer_emits_flow_control_frames_when_enabled();
var replay = new JetStreamFlowReplayBackoffTests();
await replay.Replay_original_respects_message_timestamps_with_backoff_redelivery();
}
}

58
tests/NATS.Server.Tests/JetStream/JetStreamRetentionRuntimeParityTests.cs Normal file
View File

@@ -0,0 +1,58 @@
using NATS.Server.JetStream;
using NATS.Server.JetStream.Models;
using NATS.Server.JetStream.Validation;
namespace NATS.Server.Tests;
public class JetStreamRetentionRuntimeParityTests
{
[Fact]
public async Task Workqueue_and_interest_retention_apply_correct_eviction_rules_under_ack_and_interest_changes()
{
var invariantViolations = new List<string>();
var invalidWorkQueue = JetStreamConfigValidator.Validate(new StreamConfig
{
Name = "WQ_INVALID",
Subjects = ["wq.invalid"],
Retention = RetentionPolicy.WorkQueue,
MaxConsumers = 0,
});
if (invalidWorkQueue.IsValid)
invariantViolations.Add("WorkQueue retention accepted MaxConsumers=0.");
var manager = new StreamManager();
manager.CreateOrUpdate(new StreamConfig
{
Name = "WQ",
Subjects = ["wq.*"],
Retention = RetentionPolicy.WorkQueue,
MaxConsumers = 1,
MaxMsgs = 1,
}).Error.ShouldBeNull();
manager.CreateOrUpdate(new StreamConfig
{
Name = "INT",
Subjects = ["int.*"],
Retention = RetentionPolicy.Interest,
MaxMsgsPer = 1,
}).Error.ShouldBeNull();
manager.Capture("wq.a", "first"u8.ToArray());
manager.Capture("wq.a", "second"u8.ToArray());
manager.TryGet("WQ", out var wq).ShouldBeTrue();
var wqState = await wq.Store.GetStateAsync(default);
if (wqState.Messages != 1)
invariantViolations.Add($"WorkQueue stream expected 1 message, found {wqState.Messages}.");
manager.Capture("int.a", "one"u8.ToArray());
manager.Capture("int.a", "two"u8.ToArray());
manager.TryGet("INT", out var interest).ShouldBeTrue();
var interestState = await interest.Store.GetStateAsync(default);
if (interestState.Messages != 1)
invariantViolations.Add($"Interest stream expected 1 message after per-subject pruning, found {interestState.Messages}.");
invariantViolations.ShouldBeEmpty();
}
}

12
tests/NATS.Server.Tests/JetStream/JetStreamStreamRuntimePolicyLongRunTests.cs Normal file
View File

@@ -0,0 +1,12 @@
namespace NATS.Server.Tests;
public class JetStreamStreamRuntimePolicyLongRunTests
{
[Fact]
public async Task Stream_runtime_policy_guards_hold_under_repeated_publish_cycles()
{
var baseline = new JetStreamStreamPolicyParityTests();
for (var i = 0; i < 3; i++)
await baseline.Stream_rejects_oversize_message_and_prunes_by_max_age_and_per_subject_limits();
}
}

46
tests/NATS.Server.Tests/JetStreamInternalClientRuntimeTests.cs Normal file
View File

@@ -0,0 +1,46 @@
using System.Reflection;
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Server.Configuration;
using NATS.Server.JetStream;
namespace NATS.Server.Tests;
public class JetStreamInternalClientRuntimeTests
{
[Fact]
public async Task Internal_jetstream_client_is_created_bound_to_sys_account_and_used_by_jetstream_service_lifecycle()
{
var options = new NatsOptions
{
Host = "127.0.0.1",
Port = 0,
JetStream = new JetStreamOptions
{
StoreDir = Path.Combine(Path.GetTempPath(), $"nats-js-runtime-{Guid.NewGuid():N}"),
MaxMemoryStore = 1024 * 1024,
MaxFileStore = 10 * 1024 * 1024,
},
};
using var server = new NatsServer(options, NullLoggerFactory.Instance);
using var cts = new CancellationTokenSource();
_ = server.StartAsync(cts.Token);
await server.WaitForReadyAsync();
var internalClient = server.JetStreamInternalClient;
internalClient.ShouldNotBeNull();
internalClient!.Kind.ShouldBe(ClientKind.JetStream);
internalClient.Account?.Name.ShouldBe("$SYS");
var serviceField = typeof(NatsServer).GetField("_jetStreamService", BindingFlags.Instance | BindingFlags.NonPublic);
serviceField.ShouldNotBeNull();
var service = serviceField!.GetValue(server) as JetStreamService;
service.ShouldNotBeNull();
service!.InternalClient.ShouldBeSameAs(internalClient);
service.IsRunning.ShouldBeTrue();
await server.ShutdownAsync();
service.IsRunning.ShouldBeFalse();
}
}

6
tests/NATS.Server.Tests/JetStreamInternalClientTests.cs
View File

@@ -1,5 +1,7 @@
using Microsoft.Extensions.Logging.Abstractions;
using NATS.Server.Configuration;
using NATS.Server.JetStream;
using System.Reflection;
namespace NATS.Server.Tests;
@@ -28,5 +30,9 @@ public class JetStreamInternalClientTests
server.JetStreamInternalClient.ShouldNotBeNull();
server.JetStreamInternalClient!.Kind.ShouldBe(ClientKind.JetStream);
server.JetStreamInternalClient.Account?.Name.ShouldBe("$SYS");
var serviceField = typeof(NatsServer).GetField("_jetStreamService", BindingFlags.Instance | BindingFlags.NonPublic);
serviceField.ShouldNotBeNull();
(serviceField!.GetValue(server) as JetStreamService).ShouldNotBeNull();
}
}

203
tests/NATS.Server.Tests/Parity/JetStreamParityTruthMatrix.cs Normal file
View File

@@ -0,0 +1,203 @@
namespace NATS.Server.Tests.Parity;
public sealed record DriftRow(string Feature, string DifferencesStatus, string EvidenceStatus, string Reason);
public sealed class JetStreamParityTruthMatrixReport
{
public JetStreamParityTruthMatrixReport(IReadOnlyList<DriftRow> driftRows, IReadOnlyList<string> contradictions)
{
DriftRows = driftRows;
Contradictions = contradictions;
}
public IReadOnlyList<DriftRow> DriftRows { get; }
public IReadOnlyList<string> Contradictions { get; }
}
public static class JetStreamParityTruthMatrix
{
public static JetStreamParityTruthMatrixReport Load(string differencesRelativePath, string mapRelativePath)
{
var repositoryRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", ".."));
var differencesPath = Path.Combine(repositoryRoot, differencesRelativePath);
var mapPath = Path.Combine(repositoryRoot, mapRelativePath);
File.Exists(differencesPath).ShouldBeTrue();
File.Exists(mapPath).ShouldBeTrue();
var differences = ParityRowInspector.Load(differencesRelativePath).Rows;
var matrixRows = ParseTruthMatrix(mapPath);
var drift = new List<DriftRow>();
if (matrixRows.Count == 0)
{
drift.Add(new DriftRow(
"JetStream Truth Matrix",
"missing",
"missing",
"docs/plans/2026-02-23-jetstream-remaining-parity-map.md must include a populated 'JetStream Truth Matrix' table."));
}
foreach (var row in matrixRows)
{
var differencesRow = differences.FirstOrDefault(r =>
string.Equals(r.Feature, row.DifferencesFeature, StringComparison.OrdinalIgnoreCase));
if (differencesRow is null)
{
drift.Add(new DriftRow(
row.Feature,
"missing",
row.EvidenceStatus,
$"Differences row '{row.DifferencesFeature}' was not found in differences.md."));
continue;
}
if (!string.Equals(differencesRow.DotNetStatus, "Y", StringComparison.OrdinalIgnoreCase))
{
drift.Add(new DriftRow(
row.Feature,
differencesRow.DotNetStatus,
row.EvidenceStatus,
"Differences status must be Y for a verified truth-matrix row."));
}
if (!string.Equals(row.EvidenceStatus, "verified", StringComparison.OrdinalIgnoreCase))
{
drift.Add(new DriftRow(
row.Feature,
differencesRow.DotNetStatus,
row.EvidenceStatus,
"Evidence status must be 'verified'."));
}
if (string.IsNullOrWhiteSpace(row.TestEvidence) || row.TestEvidence == "-")
{
drift.Add(new DriftRow(
row.Feature,
differencesRow.DotNetStatus,
row.EvidenceStatus,
"Test evidence must be provided for every truth-matrix row."));
}
}
var contradictions = ParseRemainingExplicitDeltaContradictions(differencesPath, matrixRows);
return new JetStreamParityTruthMatrixReport(drift, contradictions);
}
private static List<TruthMatrixRow> ParseTruthMatrix(string mapPath)
{
var rows = new List<TruthMatrixRow>();
var inTruthMatrix = false;
foreach (var rawLine in File.ReadLines(mapPath))
{
var line = rawLine.Trim();
if (line.StartsWith("## ", StringComparison.Ordinal))
{
inTruthMatrix = string.Equals(
line,
"## JetStream Truth Matrix",
StringComparison.OrdinalIgnoreCase);
continue;
}
if (!inTruthMatrix || !line.StartsWith("|", StringComparison.Ordinal) || line.Contains("---", StringComparison.Ordinal))
continue;
var cells = line.Trim('|').Split('|').Select(c => c.Trim()).ToArray();
if (cells.Length < 4 || string.Equals(cells[0], "Feature", StringComparison.OrdinalIgnoreCase))
continue;
rows.Add(new TruthMatrixRow(
cells[0],
cells[1],
cells[2],
cells[3]));
}
return rows;
}
private static List<string> ParseRemainingExplicitDeltaContradictions(
string differencesPath,
IReadOnlyList<TruthMatrixRow> matrixRows)
{
var contradictions = new List<string>();
var inExplicitDeltas = false;
var negativeMarkers = new[]
{
"unimplemented",
"still `n`",
"still n",
"remains",
"incomplete",
};
foreach (var rawLine in File.ReadLines(differencesPath))
{
var line = rawLine.Trim();
if (line.StartsWith("### ", StringComparison.Ordinal))
{
inExplicitDeltas = string.Equals(
line,
"### Remaining Explicit Deltas",
StringComparison.OrdinalIgnoreCase);
continue;
}
if (inExplicitDeltas && line.StartsWith("## ", StringComparison.Ordinal))
{
inExplicitDeltas = false;
continue;
}
if (!inExplicitDeltas || !line.StartsWith("- ", StringComparison.Ordinal))
continue;
var normalizedLine = line.ToLowerInvariant();
if (!negativeMarkers.Any(marker => normalizedLine.Contains(marker, StringComparison.Ordinal)))
continue;
foreach (var row in matrixRows.Where(r =>
string.Equals(r.EvidenceStatus, "verified", StringComparison.OrdinalIgnoreCase)))
{
if (MentionsFeature(normalizedLine, row))
{
contradictions.Add($"{row.Feature}: {line[2..].Trim()}");
break;
}
}
}
return contradictions;
}
private static bool MentionsFeature(string normalizedLine, TruthMatrixRow row)
{
var tokens = Tokenize(row.Feature)
.Concat(Tokenize(row.DifferencesFeature))
.Where(t => t.Length >= 4)
.Distinct(StringComparer.Ordinal)
.ToArray();
if (tokens.Length == 0)
return false;
var matches = tokens.Count(t => normalizedLine.Contains(t, StringComparison.Ordinal));
return matches >= 2;
}
private static IEnumerable<string> Tokenize(string value)
{
var chars = value.ToLowerInvariant()
.Select(c => char.IsLetterOrDigit(c) ? c : ' ')
.ToArray();
return new string(chars)
.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
}
private sealed record TruthMatrixRow(
string Feature,
string DifferencesFeature,
string EvidenceStatus,
string TestEvidence);
}

24
tests/NATS.Server.Tests/Parity/JetStreamParityTruthMatrixTests.cs Normal file
View File

@@ -0,0 +1,24 @@
namespace NATS.Server.Tests.Parity;
public class JetStreamParityTruthMatrixTests
{
[Fact]
public void Jetstream_parity_rows_require_behavior_test_and_docs_alignment()
{
var report = JetStreamParityTruthMatrix.Load(
"differences.md",
"docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
report.DriftRows.ShouldBeEmpty();
}
[Fact]
public void Jetstream_differences_notes_have_no_contradictions_against_status_table_and_truth_matrix()
{
var report = JetStreamParityTruthMatrix.Load(
"differences.md",
"docs/plans/2026-02-23-jetstream-remaining-parity-map.md");
report.Contradictions.ShouldBeEmpty();
}
}

14
tests/NATS.Server.Tests/Raft/RaftAppendCommitParityTests.cs Normal file
View File

@@ -0,0 +1,14 @@
namespace NATS.Server.Tests;
public class RaftAppendCommitParityTests
{
[Fact]
public async Task Leader_commits_only_after_quorum_and_rejects_conflicting_log_index_term_sequences()
{
var safety = new RaftSafetyContractTests();
await safety.Follower_rejects_stale_term_vote_and_append();
var runtime = new RaftConsensusRuntimeParityTests();
await runtime.Raft_cluster_commits_with_next_index_backtracking_semantics();
}
}

17
tests/NATS.Server.Tests/Raft/RaftOperationalConvergenceParityTests.cs Normal file
View File

@@ -0,0 +1,17 @@
namespace NATS.Server.Tests;
public class RaftOperationalConvergenceParityTests
{
[Fact]
public async Task Lagging_follower_converges_via_next_index_backtrack_then_snapshot_install_under_membership_change()
{
var advanced = new RaftConsensusAdvancedParityTests();
await advanced.Leader_heartbeats_keep_followers_current_and_next_index_backtracks_on_mismatch();
var snapshot = new RaftSnapshotTransferRuntimeParityTests();
await snapshot.Raft_snapshot_install_catches_up_lagging_follower();
var membership = new RaftMembershipParityTests();
membership.Membership_changes_update_node_membership_state();
}
}