feat(batch11): complete filestore init feature and test port
This commit is contained in:
Joseph Doherty
@@ -81,6 +81,59 @@ public sealed partial class ConcurrencyTests1
|
||||
});
|
||||
}
|
||||
|
||||
[Fact] // T:2427
|
||||
public void NoRaceJetStreamFileStoreKeyFileCleanup_ShouldSucceed()
|
||||
{
|
||||
WithStore((_, root) =>
|
||||
{
|
||||
var msgDir = Path.Combine(root, FileStoreDefaults.MsgDir);
|
||||
Directory.CreateDirectory(msgDir);
|
||||
var perm = UnixFileMode.UserRead | UnixFileMode.UserWrite;
|
||||
|
||||
var errors = new ConcurrentQueue<Exception>();
|
||||
Parallel.For(0, 300, i =>
|
||||
{
|
||||
var payload = BitConverter.GetBytes(i);
|
||||
var keyFile = Path.Combine(msgDir, string.Format(FileStoreDefaults.KeyScan, (uint)(i + 1)));
|
||||
var err = JetStreamFileStore.WriteAtomically(keyFile, payload, perm, sync: true);
|
||||
if (err != null)
|
||||
errors.Enqueue(err);
|
||||
});
|
||||
|
||||
errors.ShouldBeEmpty();
|
||||
var keyFiles = Directory.GetFiles(msgDir, "*.key");
|
||||
keyFiles.Length.ShouldBe(300);
|
||||
|
||||
foreach (var key in keyFiles.Skip(1))
|
||||
File.Delete(key);
|
||||
|
||||
Directory.GetFiles(msgDir, "*.key").Length.ShouldBe(1);
|
||||
Directory.GetFiles(msgDir, "*.tmp").ShouldBeEmpty();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact] // T:2447
|
||||
public void NoRaceEncodeConsumerStateBug_ShouldSucceed()
|
||||
{
|
||||
for (var i = 0; i < 5_000; i++)
|
||||
{
|
||||
var pending = new Pending
|
||||
{
|
||||
Sequence = 1,
|
||||
Timestamp = DateTimeOffset.UtcNow.AddSeconds(1).ToUnixTimeSeconds() * 1_000_000_000L,
|
||||
};
|
||||
var state = new ConsumerState
|
||||
{
|
||||
Delivered = new SequencePair { Consumer = 1, Stream = 1 },
|
||||
Pending = new Dictionary<ulong, Pending> { [1] = pending },
|
||||
};
|
||||
|
||||
var encoded = StoreParity.EncodeConsumerState(state);
|
||||
var (_, err) = JetStreamFileStore.DecodeConsumerState(encoded);
|
||||
err.ShouldBeNull();
|
||||
}
|
||||
}
|
||||
|
||||
private static void WithStore(Action<JetStreamFileStore, string> action, StreamConfig? cfg = null)
|
||||
{
|
||||
var root = NewRoot();
|
||||
|
||||
@@ -910,4 +910,278 @@ public sealed partial class JetStreamFileStoreTests
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact] // T:385
|
||||
public void FileStoreConsumer_ShouldSucceed()
|
||||
{
|
||||
WithStore((fs, _) =>
|
||||
{
|
||||
var consumer = fs.ConsumerStore("obs22", DateTime.UtcNow, new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit });
|
||||
var (initial, initialErr) = consumer.State();
|
||||
initialErr.ShouldBeNull();
|
||||
initial.ShouldNotBeNull();
|
||||
initial!.Delivered.Consumer.ShouldBe(0UL);
|
||||
|
||||
var state = new ConsumerState
|
||||
{
|
||||
Delivered = new SequencePair { Consumer = 100, Stream = 122 },
|
||||
AckFloor = new SequencePair { Consumer = 50, Stream = 72 },
|
||||
Pending = new Dictionary<ulong, Pending>
|
||||
{
|
||||
[75] = new() { Sequence = 75, Timestamp = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L },
|
||||
[80] = new() { Sequence = 80, Timestamp = DateTimeOffset.UtcNow.AddSeconds(1).ToUnixTimeSeconds() * 1_000_000_000L },
|
||||
},
|
||||
Redelivered = new Dictionary<ulong, ulong>
|
||||
{
|
||||
[90] = 2,
|
||||
},
|
||||
};
|
||||
|
||||
consumer.Update(state);
|
||||
var (updated, updateErr) = consumer.State();
|
||||
updateErr.ShouldBeNull();
|
||||
updated.ShouldNotBeNull();
|
||||
updated!.Delivered.Consumer.ShouldBe(state.Delivered.Consumer);
|
||||
updated.Delivered.Stream.ShouldBe(state.Delivered.Stream);
|
||||
updated.AckFloor.Consumer.ShouldBe(state.AckFloor.Consumer);
|
||||
updated.AckFloor.Stream.ShouldBe(state.AckFloor.Stream);
|
||||
updated.Pending!.Count.ShouldBe(2);
|
||||
updated.Redelivered!.Count.ShouldBe(1);
|
||||
|
||||
state.AckFloor = new SequencePair { Consumer = 200, Stream = 100 };
|
||||
Should.Throw<InvalidOperationException>(() => consumer.Update(state));
|
||||
|
||||
consumer.Stop();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact] // T:386
|
||||
public void FileStoreConsumerEncodeDecodeRedelivered_ShouldSucceed()
|
||||
{
|
||||
var state = new ConsumerState
|
||||
{
|
||||
Delivered = new SequencePair { Consumer = 100, Stream = 100 },
|
||||
AckFloor = new SequencePair { Consumer = 50, Stream = 50 },
|
||||
Redelivered = new Dictionary<ulong, ulong>
|
||||
{
|
||||
[122] = 3,
|
||||
[144] = 8,
|
||||
},
|
||||
};
|
||||
|
||||
var buf = StoreParity.EncodeConsumerState(state);
|
||||
var (decoded, err) = JetStreamFileStore.DecodeConsumerState(buf);
|
||||
err.ShouldBeNull();
|
||||
decoded.ShouldNotBeNull();
|
||||
decoded!.Delivered.Consumer.ShouldBe(state.Delivered.Consumer);
|
||||
decoded.Delivered.Stream.ShouldBe(state.Delivered.Stream);
|
||||
decoded.AckFloor.Consumer.ShouldBe(state.AckFloor.Consumer);
|
||||
decoded.AckFloor.Stream.ShouldBe(state.AckFloor.Stream);
|
||||
decoded.Redelivered.ShouldNotBeNull();
|
||||
decoded.Redelivered![122].ShouldBe(3UL);
|
||||
decoded.Redelivered[144].ShouldBe(8UL);
|
||||
}
|
||||
|
||||
[Fact] // T:387
|
||||
public void FileStoreConsumerEncodeDecodePendingBelowStreamAckFloor_ShouldSucceed()
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L;
|
||||
var state = new ConsumerState
|
||||
{
|
||||
Delivered = new SequencePair { Consumer = 1192, Stream = 10185 },
|
||||
AckFloor = new SequencePair { Consumer = 1189, Stream = 10815 },
|
||||
Pending = new Dictionary<ulong, Pending>
|
||||
{
|
||||
[10782] = new() { Sequence = 1190, Timestamp = now },
|
||||
[10810] = new() { Sequence = 1191, Timestamp = now + 1_000_000_000L },
|
||||
[10815] = new() { Sequence = 1192, Timestamp = now + 2_000_000_000L },
|
||||
},
|
||||
};
|
||||
|
||||
var buf = StoreParity.EncodeConsumerState(state);
|
||||
var (decoded, err) = JetStreamFileStore.DecodeConsumerState(buf);
|
||||
err.ShouldBeNull();
|
||||
decoded.ShouldNotBeNull();
|
||||
decoded!.Pending.ShouldNotBeNull();
|
||||
decoded.Pending.Count.ShouldBe(3);
|
||||
decoded.Pending.ContainsKey(10782).ShouldBeTrue();
|
||||
decoded.Pending.ContainsKey(10810).ShouldBeTrue();
|
||||
decoded.Pending.ContainsKey(10815).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact] // T:393
|
||||
public void FileStoreConsumerRedeliveredLost_ShouldSucceed()
|
||||
{
|
||||
var root = NewRoot();
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
try
|
||||
{
|
||||
var fs = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig());
|
||||
var cfg = new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit };
|
||||
var consumer = fs.ConsumerStore("o22", DateTime.UtcNow, cfg);
|
||||
var ts = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L;
|
||||
|
||||
consumer.UpdateDelivered(1, 1, 1, ts);
|
||||
consumer.UpdateDelivered(2, 1, 2, ts);
|
||||
consumer.UpdateDelivered(3, 1, 3, ts);
|
||||
consumer.UpdateDelivered(4, 1, 4, ts);
|
||||
consumer.UpdateDelivered(5, 2, 1, ts);
|
||||
consumer.Stop();
|
||||
|
||||
consumer = fs.ConsumerStore("o22", DateTime.UtcNow, cfg);
|
||||
var (state, err) = consumer.State();
|
||||
err.ShouldBeNull();
|
||||
state.ShouldNotBeNull();
|
||||
state!.Redelivered.ShouldNotBeNull();
|
||||
state.Redelivered.Count.ShouldBeGreaterThan(0);
|
||||
|
||||
consumer.UpdateAcks(2, 1);
|
||||
consumer.UpdateAcks(5, 2);
|
||||
|
||||
var (afterAcks, afterErr) = consumer.State();
|
||||
afterErr.ShouldBeNull();
|
||||
afterAcks.ShouldNotBeNull();
|
||||
afterAcks!.Pending.ShouldBeNull();
|
||||
afterAcks.Redelivered.ShouldBeNull();
|
||||
|
||||
consumer.Stop();
|
||||
fs.Stop();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact] // T:395
|
||||
public void FileStoreConsumerDeliveredUpdates_ShouldSucceed()
|
||||
{
|
||||
WithStore((fs, _) =>
|
||||
{
|
||||
var consumer = fs.ConsumerStore("o22", DateTime.UtcNow, new ConsumerConfig());
|
||||
|
||||
var ts = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L;
|
||||
consumer.UpdateDelivered(1, 100, 1, ts);
|
||||
consumer.UpdateDelivered(2, 110, 1, ts);
|
||||
consumer.UpdateDelivered(5, 130, 1, ts);
|
||||
|
||||
var (state, err) = consumer.State();
|
||||
err.ShouldBeNull();
|
||||
state.ShouldNotBeNull();
|
||||
state!.Delivered.Consumer.ShouldBe(5UL);
|
||||
state.Delivered.Stream.ShouldBe(130UL);
|
||||
state.AckFloor.Consumer.ShouldBe(5UL);
|
||||
state.AckFloor.Stream.ShouldBe(130UL);
|
||||
state.Pending.ShouldBeNull();
|
||||
|
||||
Should.Throw<InvalidOperationException>(() => consumer.UpdateAcks(1, 100));
|
||||
Should.Throw<InvalidOperationException>(() => consumer.UpdateDelivered(6, 131, 2, ts));
|
||||
|
||||
consumer.Stop();
|
||||
});
|
||||
}
|
||||
|
||||
[Fact] // T:396
|
||||
public void FileStoreConsumerDeliveredAndAckUpdates_ShouldSucceed()
|
||||
{
|
||||
var root = NewRoot();
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
try
|
||||
{
|
||||
var fs = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig());
|
||||
var cfg = new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit };
|
||||
var consumer = fs.ConsumerStore("o22", DateTime.UtcNow, cfg);
|
||||
var ts = DateTimeOffset.UtcNow.ToUnixTimeSeconds() * 1_000_000_000L;
|
||||
|
||||
consumer.UpdateDelivered(1, 100, 1, ts);
|
||||
consumer.UpdateDelivered(2, 110, 1, ts);
|
||||
consumer.UpdateDelivered(3, 130, 1, ts);
|
||||
consumer.UpdateDelivered(4, 150, 1, ts);
|
||||
consumer.UpdateDelivered(5, 165, 1, ts);
|
||||
|
||||
var (beforeAcks, beforeErr) = consumer.State();
|
||||
beforeErr.ShouldBeNull();
|
||||
beforeAcks.ShouldNotBeNull();
|
||||
beforeAcks!.Pending!.Count.ShouldBe(5);
|
||||
|
||||
Should.Throw<Exception>(() => consumer.UpdateAcks(3, 101));
|
||||
Should.Throw<Exception>(() => consumer.UpdateAcks(1, 1));
|
||||
|
||||
consumer.UpdateAcks(1, 100);
|
||||
consumer.UpdateAcks(2, 110);
|
||||
consumer.UpdateAcks(3, 130);
|
||||
var (afterAcks, afterErr) = consumer.State();
|
||||
afterErr.ShouldBeNull();
|
||||
afterAcks.ShouldNotBeNull();
|
||||
afterAcks!.Pending!.Count.ShouldBe(2);
|
||||
|
||||
consumer.Stop();
|
||||
consumer = fs.ConsumerStore("o22", DateTime.UtcNow, cfg);
|
||||
var (restored, restoredErr) = consumer.State();
|
||||
restoredErr.ShouldBeNull();
|
||||
restored.ShouldNotBeNull();
|
||||
restored!.Pending!.Count.ShouldBe(2);
|
||||
|
||||
consumer.Stop();
|
||||
fs.Stop();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact] // T:402
|
||||
public void FileStoreBadConsumerState_ShouldSucceed()
|
||||
{
|
||||
var bs = new byte[]
|
||||
{
|
||||
0x16, 0x02, 0x01, 0x01, 0x03, 0x02, 0x01, 0x98, 0xF4, 0x8A,
|
||||
0x8A, 0x0C, 0x01, 0x03, 0x86, 0xFA, 0x0A, 0x01, 0x00, 0x01,
|
||||
};
|
||||
|
||||
var (state, err) = JetStreamFileStore.DecodeConsumerState(bs);
|
||||
err.ShouldBeNull();
|
||||
state.ShouldNotBeNull();
|
||||
}
|
||||
|
||||
[Fact] // T:440
|
||||
public void FileStoreConsumerStoreEncodeAfterRestart_ShouldSucceed()
|
||||
{
|
||||
var root = NewRoot();
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
try
|
||||
{
|
||||
var persisted = new ConsumerState
|
||||
{
|
||||
Delivered = new SequencePair { Consumer = 22, Stream = 22 },
|
||||
AckFloor = new SequencePair { Consumer = 11, Stream = 11 },
|
||||
};
|
||||
|
||||
var fs1 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig());
|
||||
var c1 = fs1.ConsumerStore("o22", DateTime.UtcNow, new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit });
|
||||
c1.Update(persisted);
|
||||
c1.Stop();
|
||||
fs1.Stop();
|
||||
|
||||
var fs2 = JetStreamFileStore.NewFileStore(new FileStoreConfig { StoreDir = root }, DefaultStreamConfig());
|
||||
var c2 = fs2.ConsumerStore("o22", DateTime.UtcNow, new ConsumerConfig { AckPolicy = AckPolicy.AckExplicit });
|
||||
var (state, err) = c2.State();
|
||||
err.ShouldBeNull();
|
||||
state.ShouldNotBeNull();
|
||||
state!.Delivered.Consumer.ShouldBe(persisted.Delivered.Consumer);
|
||||
state.Delivered.Stream.ShouldBe(persisted.Delivered.Stream);
|
||||
state.AckFloor.Consumer.ShouldBe(persisted.AckFloor.Consumer);
|
||||
state.AckFloor.Stream.ShouldBe(persisted.AckFloor.Stream);
|
||||
c2.Stop();
|
||||
fs2.Stop();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(root, recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user