diff --git a/README.md b/README.md index 3f4408a..b29ae75 100755 --- a/README.md +++ b/README.md @@ -104,6 +104,27 @@ Operational procedures, diagnostics, and escalation are documented in: - [`docs/runbook.md`](docs/runbook.md) - [`docs/troubleshooting.md`](docs/troubleshooting.md) +## Checkpoint Modes + +`DocumentDbContext` and `StorageEngine` support explicit checkpoint modes via `CheckpointMode`: + +- `Passive`: non-blocking; skips if checkpoint lock is contended. +- `Full`: applies committed WAL pages and appends a checkpoint marker, without truncating WAL. +- `Truncate`: applies committed WAL pages and truncates WAL. +- `Restart`: truncate + WAL writer restart. + +Example: + +```csharp +using ZB.MOM.WW.CBDD.Core.Transactions; + +var result = db.Checkpoint(CheckpointMode.Full); +if (!result.Executed) +{ + // passive checkpoint can be skipped under contention +} +``` + ## Security And Compliance Posture - CBDD relies on host and process-level access controls. diff --git a/docs/features/storage-transactions.md b/docs/features/storage-transactions.md index 88a027b..5d0eb85 100644 --- a/docs/features/storage-transactions.md +++ b/docs/features/storage-transactions.md @@ -29,6 +29,24 @@ Non-goals: - `WriteAheadLog` - Storage engine modules under `src/CBDD.Core/Storage` +Checkpoint APIs: +- `DocumentDbContext.Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)` +- `DocumentDbContext.CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)` +- `StorageEngine.Checkpoint(CheckpointMode mode)` +- `StorageEngine.CheckpointAsync(CheckpointMode mode, CancellationToken ct = default)` + +Checkpoint modes: +- `Passive`: non-blocking best-effort checkpoint. Returns `Executed = false` when lock is contended. +- `Full`: applies committed WAL pages and appends a WAL checkpoint marker without truncating WAL. +- `Truncate`: applies committed WAL pages and truncates WAL (default behavior). +- `Restart`: same as truncate, then reinitializes WAL writer session. + +Usage guidance: +- Use `Passive` for background/low-priority maintenance where latency matters more than immediate WAL cleanup. +- Use `Full` when you want durable page-file sync but prefer to preserve WAL history until a later truncate. +- Use `Truncate` for routine manual checkpoints and disk-space recovery. +- Use `Restart` for aggressive maintenance boundaries (for example after incident remediation flows). + ## Permissions And Data Handling - Database files require host-managed filesystem access controls. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2317698..8753d4e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -28,6 +28,8 @@ Database startup fails or recovery path throws WAL/storage errors. ### Resolution - Pin consumers to last known-good package. - Apply fix and add regression coverage in recovery/transaction tests. +- If WAL growth is the issue, run `CheckpointMode.Truncate` (or `Restart`) instead of `Full`. +- If foreground latency is a concern, schedule `CheckpointMode.Passive` retries and use `Truncate` during maintenance windows. ## Query And Index Issues diff --git a/src/CBDD.Bson/BsonDocument.cs b/src/CBDD.Bson/Document/BsonDocument.cs similarity index 100% rename from src/CBDD.Bson/BsonDocument.cs rename to src/CBDD.Bson/Document/BsonDocument.cs diff --git a/src/CBDD.Bson/BsonType.cs b/src/CBDD.Bson/Document/BsonType.cs similarity index 100% rename from src/CBDD.Bson/BsonType.cs rename to src/CBDD.Bson/Document/BsonType.cs diff --git a/src/CBDD.Bson/BsonBufferWriter.cs b/src/CBDD.Bson/IO/BsonBufferWriter.cs similarity index 100% rename from src/CBDD.Bson/BsonBufferWriter.cs rename to src/CBDD.Bson/IO/BsonBufferWriter.cs diff --git a/src/CBDD.Bson/BsonSpanReader.cs b/src/CBDD.Bson/IO/BsonSpanReader.cs similarity index 100% rename from src/CBDD.Bson/BsonSpanReader.cs rename to src/CBDD.Bson/IO/BsonSpanReader.cs diff --git a/src/CBDD.Bson/BsonSpanWriter.cs b/src/CBDD.Bson/IO/BsonSpanWriter.cs similarity index 100% rename from src/CBDD.Bson/BsonSpanWriter.cs rename to src/CBDD.Bson/IO/BsonSpanWriter.cs diff --git a/src/CBDD.Bson/Attributes.cs b/src/CBDD.Bson/Metadata/Attributes.cs similarity index 100% rename from src/CBDD.Bson/Attributes.cs rename to src/CBDD.Bson/Metadata/Attributes.cs diff --git a/src/CBDD.Bson/ObjectId.cs b/src/CBDD.Bson/Types/ObjectId.cs similarity index 100% rename from src/CBDD.Bson/ObjectId.cs rename to src/CBDD.Bson/Types/ObjectId.cs diff --git a/src/CBDD.Core/DocumentDbContext.cs b/src/CBDD.Core/Context/DocumentDbContext.cs similarity index 91% rename from src/CBDD.Core/DocumentDbContext.cs rename to src/CBDD.Core/Context/DocumentDbContext.cs index 207e43e..4cbbefa 100755 --- a/src/CBDD.Core/DocumentDbContext.cs +++ b/src/CBDD.Core/Context/DocumentDbContext.cs @@ -312,10 +312,10 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde /// Commits the current transaction asynchronously if one is active. /// /// The cancellation token. - public async Task SaveChangesAsync(CancellationToken ct = default) - { - if (_disposed) - throw new ObjectDisposedException(nameof(DocumentDbContext)); + public async Task SaveChangesAsync(CancellationToken ct = default) + { + if (_disposed) + throw new ObjectDisposedException(nameof(DocumentDbContext)); if (CurrentTransaction != null) { try @@ -325,13 +325,40 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde finally { CurrentTransaction = null; - } - } - } - - /// - /// Returns a point-in-time snapshot of compression telemetry counters. - /// + } + } + } + + /// + /// Executes a checkpoint using the requested mode. + /// + /// Checkpoint mode to execute. + /// The checkpoint execution result. + public CheckpointResult Checkpoint(CheckpointMode mode = CheckpointMode.Truncate) + { + if (_disposed) + throw new ObjectDisposedException(nameof(DocumentDbContext)); + + return Engine.Checkpoint(mode); + } + + /// + /// Executes a checkpoint asynchronously using the requested mode. + /// + /// Checkpoint mode to execute. + /// The cancellation token. + /// The checkpoint execution result. + public Task CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default) + { + if (_disposed) + throw new ObjectDisposedException(nameof(DocumentDbContext)); + + return Engine.CheckpointAsync(mode, ct); + } + + /// + /// Returns a point-in-time snapshot of compression telemetry counters. + /// public CompressionStats GetCompressionStats() { if (_disposed) diff --git a/src/CBDD.Core/Query/BsonExpressionEvaluator.cs b/src/CBDD.Core/Query/BsonExpressionEvaluator.cs index f1cdb07..73abac2 100755 --- a/src/CBDD.Core/Query/BsonExpressionEvaluator.cs +++ b/src/CBDD.Core/Query/BsonExpressionEvaluator.cs @@ -3,16 +3,16 @@ using ZB.MOM.WW.CBDD.Bson; namespace ZB.MOM.WW.CBDD.Core.Query; -internal static class BsonExpressionEvaluator -{ - /// - /// Attempts to compile a LINQ predicate expression into a BSON reader predicate. - /// - /// The entity type of the original expression. - /// The lambda expression to compile. - /// A compiled predicate when supported; otherwise, . - public static Func? TryCompile(LambdaExpression expression) - { +internal static class BsonExpressionEvaluator +{ + /// + /// Attempts to compile a LINQ predicate expression into a BSON reader predicate. + /// + /// The entity type of the original expression. + /// The lambda expression to compile. + /// A compiled predicate when supported; otherwise, . + public static Func? TryCompile(LambdaExpression expression) + { // Simple optimization for: x => x.Prop op Constant if (expression.Body is BinaryExpression binary) { @@ -34,10 +34,10 @@ internal static class BsonExpressionEvaluator if (member.Expression == expression.Parameters[0]) { var propertyName = member.Member.Name.ToLowerInvariant(); - var value = constant.Value; - - // Handle Id mapping? - // If property is "id", Bson field is "_id" + var value = constant.Value; + + // Handle Id mapping? + // If property is "id", Bson field is "_id" if (propertyName == "id") propertyName = "_id"; return CreatePredicate(propertyName, value, nodeType); @@ -58,31 +58,31 @@ internal static class BsonExpressionEvaluator }; private static Func? CreatePredicate(string propertyName, object? targetValue, ExpressionType op) - { - // We need to return a delegate that searches for propertyName in BsonSpanReader and compares - - return reader => + { + // We need to return a delegate that searches for propertyName in BsonSpanReader and compares + + return reader => { - try + try { reader.ReadDocumentSize(); while (reader.Remaining > 0) { var type = reader.ReadBsonType(); - if (type == 0) break; - - var name = reader.ReadElementHeader(); - + if (type == 0) break; + + var name = reader.ReadElementHeader(); + if (name == propertyName) { - // Found it! Read value and compare + // Found -> read value and compare return Compare(ref reader, type, targetValue, op); - } - + } + reader.SkipValue(type); } } - catch + catch { return false; } @@ -91,10 +91,10 @@ internal static class BsonExpressionEvaluator } private static bool Compare(ref BsonSpanReader reader, BsonType type, object? target, ExpressionType op) - { - // This is complex because we need to handle types. - // For MVP, handle Int32, String, ObjectId - + { + // This is complex because we need to handle types. + // For MVP, handle Int32, String, ObjectId + if (type == BsonType.Int32) { var val = reader.ReadInt32(); @@ -132,12 +132,12 @@ internal static class BsonExpressionEvaluator } else if (type == BsonType.ObjectId && target is ObjectId targetId) { - var val = reader.ReadObjectId(); - // ObjectId only supports Equal check easily unless we implement complex logic + var val = reader.ReadObjectId(); + // ObjectId only supports Equal check easily unless we implement complex logic if (op == ExpressionType.Equal) return val.Equals(targetId); if (op == ExpressionType.NotEqual) return !val.Equals(targetId); - } - + } + return false; } } diff --git a/src/CBDD.Core/Storage/StorageEngine.Recovery.cs b/src/CBDD.Core/Storage/StorageEngine.Recovery.cs index ce68227..ba3a7d2 100755 --- a/src/CBDD.Core/Storage/StorageEngine.Recovery.cs +++ b/src/CBDD.Core/Storage/StorageEngine.Recovery.cs @@ -2,8 +2,8 @@ using ZB.MOM.WW.CBDD.Core.Transactions; namespace ZB.MOM.WW.CBDD.Core.Storage; -public sealed partial class StorageEngine -{ +public sealed partial class StorageEngine +{ /// /// Gets the current size of the WAL file. /// @@ -29,151 +29,244 @@ public sealed partial class StorageEngine _wal.Flush(); } - /// - /// Performs a checkpoint: merges WAL into PageFile. - /// Uses in-memory WAL index for efficiency and consistency. - /// - /// - /// Performs a checkpoint: merges WAL into PageFile. - /// Uses in-memory WAL index for efficiency and consistency. - /// - public void Checkpoint() - { - _commitLock.Wait(); - try - { - CheckpointInternal(); - } - finally - { - _commitLock.Release(); - } - } - - private void CheckpointInternal() + /// + /// Performs a truncate checkpoint by default. + /// + public void Checkpoint() { - if (_walIndex.IsEmpty) + _ = Checkpoint(CheckpointMode.Truncate); + } + + /// + /// Performs a checkpoint using the requested mode. + /// + /// Checkpoint mode to execute. + /// The checkpoint execution result. + public CheckpointResult Checkpoint(CheckpointMode mode) + { + bool lockAcquired; + if (mode == CheckpointMode.Passive) { - // WAL may still contain begin/commit records for read-only transactions. + lockAcquired = _commitLock.Wait(0); + if (!lockAcquired) + { + var walSize = _wal.GetCurrentSize(); + return new CheckpointResult(mode, false, 0, walSize, walSize, false, false); + } + } + else + { + _commitLock.Wait(); + lockAcquired = true; + } + + try + { + return CheckpointInternal(mode); + } + finally + { + if (lockAcquired) + { + _commitLock.Release(); + } + } + } + + private void CheckpointInternal() + => _ = CheckpointInternal(CheckpointMode.Truncate); + + private CheckpointResult CheckpointInternal(CheckpointMode mode) + { + var walBytesBefore = _wal.GetCurrentSize(); + var appliedPages = 0; + var truncated = false; + var restarted = false; + + // 1. Write all committed pages from index to PageFile. + foreach (var kvp in _walIndex) + { + _pageFile.WritePage(kvp.Key, kvp.Value); + appliedPages++; + } + + // 2. Flush PageFile to ensure durability. + if (appliedPages > 0) + { + _pageFile.Flush(); + } + + // 3. Clear in-memory WAL index (now persisted). + _walIndex.Clear(); + + // 4. Apply mode-specific WAL handling. + switch (mode) + { + case CheckpointMode.Passive: + case CheckpointMode.Full: + if (walBytesBefore > 0 || appliedPages > 0) + { + _wal.WriteCheckpointRecord(); + _wal.Flush(); + } + break; + case CheckpointMode.Truncate: + if (walBytesBefore > 0) + { + _wal.Truncate(); + truncated = true; + } + break; + case CheckpointMode.Restart: + _wal.Restart(); + truncated = true; + restarted = true; + break; + default: + throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported checkpoint mode."); + } + + var walBytesAfter = _wal.GetCurrentSize(); + return new CheckpointResult(mode, true, appliedPages, walBytesBefore, walBytesAfter, truncated, restarted); + } + + /// + /// Performs a truncate checkpoint asynchronously by default. + /// + /// The cancellation token. + public async Task CheckpointAsync(CancellationToken ct = default) + { + _ = await CheckpointAsync(CheckpointMode.Truncate, ct); + } + + /// + /// Performs a checkpoint asynchronously using the requested mode. + /// + /// Checkpoint mode to execute. + /// The cancellation token. + /// A task that represents the asynchronous checkpoint operation. + public async Task CheckpointAsync(CheckpointMode mode, CancellationToken ct = default) + { + bool lockAcquired; + if (mode == CheckpointMode.Passive) + { + lockAcquired = await _commitLock.WaitAsync(0, ct); + if (!lockAcquired) + { + var walSize = _wal.GetCurrentSize(); + return new CheckpointResult(mode, false, 0, walSize, walSize, false, false); + } + } + else + { + await _commitLock.WaitAsync(ct); + lockAcquired = true; + } + + try + { + // Checkpoint work is synchronous over MMF/page writes for now. + return CheckpointInternal(mode); + } + finally + { + if (lockAcquired) + { + _commitLock.Release(); + } + } + } + + /// + /// Recovers from crash by replaying WAL. + /// Applies committed transactions to PageFile in deterministic WAL order, then truncates WAL. + /// + public void Recover() + { + _commitLock.Wait(); + try + { + // 1. Read WAL and locate the latest checkpoint boundary. + var records = _wal.ReadAll(); + var startIndex = 0; + for (var i = records.Count - 1; i >= 0; i--) + { + if (records[i].Type == WalRecordType.Checkpoint) + { + startIndex = i + 1; + break; + } + } + + // 2. Replay WAL in source order with deterministic commit application. + var pendingWrites = new Dictionary>(); + var appliedAny = false; + + for (var i = startIndex; i < records.Count; i++) + { + var record = records[i]; + switch (record.Type) + { + case WalRecordType.Begin: + if (!pendingWrites.ContainsKey(record.TransactionId)) + { + pendingWrites[record.TransactionId] = new List<(uint, byte[])>(); + } + break; + case WalRecordType.Write: + if (record.AfterImage == null) + { + break; + } + + if (!pendingWrites.TryGetValue(record.TransactionId, out var writes)) + { + writes = new List<(uint, byte[])>(); + pendingWrites[record.TransactionId] = writes; + } + + writes.Add((record.PageId, record.AfterImage)); + break; + case WalRecordType.Commit: + if (!pendingWrites.TryGetValue(record.TransactionId, out var committedWrites)) + { + break; + } + + foreach (var (pageId, data) in committedWrites) + { + _pageFile.WritePage(pageId, data); + appliedAny = true; + } + + pendingWrites.Remove(record.TransactionId); + break; + case WalRecordType.Abort: + pendingWrites.Remove(record.TransactionId); + break; + case WalRecordType.Checkpoint: + pendingWrites.Clear(); + break; + } + } + + // 3. Flush PageFile to ensure durability. + if (appliedAny) + { + _pageFile.Flush(); + } + + // 4. Clear in-memory WAL index (redundant since we just recovered). + _walIndex.Clear(); + + // 5. Truncate WAL (all changes now in PageFile). if (_wal.GetCurrentSize() > 0) { _wal.Truncate(); } - return; } - - // 1. Write all committed pages from index to PageFile - foreach (var kvp in _walIndex) - { - _pageFile.WritePage(kvp.Key, kvp.Value); - } - - // 2. Flush PageFile to ensure durability - _pageFile.Flush(); - - // 3. Clear in-memory WAL index (now persisted) - _walIndex.Clear(); - - // 4. Truncate WAL (all changes now in PageFile) - _wal.Truncate(); - } - - /// - /// Performs a checkpoint asynchronously by merging WAL pages into the page file. - /// - /// The cancellation token. - /// A task that represents the asynchronous checkpoint operation. - public async Task CheckpointAsync(CancellationToken ct = default) - { - await _commitLock.WaitAsync(ct); - try + finally { - if (_walIndex.IsEmpty) - { - if (_wal.GetCurrentSize() > 0) - { - _wal.Truncate(); - } - return; - } - - // 1. Write all committed pages from index to PageFile - // PageFile writes are sync (MMF), but that's fine as per plan (ValueTask strategy for MMF) - foreach (var kvp in _walIndex) - { - _pageFile.WritePage(kvp.Key, kvp.Value); - } - - // 2. Flush PageFile to ensure durability - _pageFile.Flush(); - - // 3. Clear in-memory WAL index (now persisted) - _walIndex.Clear(); - - // 4. Truncate WAL (all changes now in PageFile) - // WAL truncation involves file resize and flush - // TODO: Add TruncateAsync to WAL? For now Truncate is sync. - _wal.Truncate(); - } - finally - { - _commitLock.Release(); - } - } - - /// - /// Recovers from crash by replaying WAL. - /// Applies all committed transactions to PageFile, then truncates WAL. - /// - public void Recover() - { - _commitLock.Wait(); - try - { - // 1. Read WAL and identify committed transactions - var records = _wal.ReadAll(); - var committedTxns = new HashSet(); - var txnWrites = new Dictionary>(); - - foreach (var record in records) - { - if (record.Type == WalRecordType.Commit) - committedTxns.Add(record.TransactionId); - else if (record.Type == WalRecordType.Write) - { - if (!txnWrites.ContainsKey(record.TransactionId)) - txnWrites[record.TransactionId] = new List<(uint, byte[])>(); - - if (record.AfterImage != null) - { - txnWrites[record.TransactionId].Add((record.PageId, record.AfterImage)); - } - } - } - - // 2. Apply committed transactions to PageFile - foreach (var txnId in committedTxns) - { - if (!txnWrites.ContainsKey(txnId)) - continue; - - foreach (var (pageId, data) in txnWrites[txnId]) - { - _pageFile.WritePage(pageId, data); - } - } - - // 3. Flush PageFile to ensure durability - _pageFile.Flush(); - - // 4. Clear in-memory WAL index (redundant since we just recovered) - _walIndex.Clear(); - - // 5. Truncate WAL (all changes now in PageFile) - _wal.Truncate(); - } - finally - { _commitLock.Release(); } } diff --git a/src/CBDD.Core/Transactions/CheckpointMode.cs b/src/CBDD.Core/Transactions/CheckpointMode.cs index 97b6e92..2855919 100755 --- a/src/CBDD.Core/Transactions/CheckpointMode.cs +++ b/src/CBDD.Core/Transactions/CheckpointMode.cs @@ -1,32 +1,52 @@ namespace ZB.MOM.WW.CBDD.Core.Transactions; -/// -/// Defines checkpoint modes for WAL (Write-Ahead Log) checkpointing. -/// Similar to SQLite's checkpoint strategies. -/// -public enum CheckpointMode -{ - /// - /// Passive checkpoint: Non-blocking, best-effort transfer from WAL to database. - /// Does not wait for readers or writers. May not checkpoint all frames. - /// +/// +/// Defines checkpoint modes for WAL (Write-Ahead Log) checkpointing. +/// Similar to SQLite's checkpoint strategies. +/// +public enum CheckpointMode +{ + /// + /// Passive checkpoint: non-blocking, best-effort transfer from WAL to database. + /// If the checkpoint lock is busy, the operation is skipped. + /// WAL content is preserved and a checkpoint marker is appended when work is applied. + /// Passive = 0, - /// - /// Full checkpoint: Waits for concurrent readers/writers, then checkpoints all - /// committed transactions from WAL to database. Blocks until complete. - /// + /// + /// Full checkpoint: waits for the checkpoint lock, transfers committed pages to + /// the page file, and preserves WAL content by appending a checkpoint marker. + /// Full = 1, - /// - /// Truncate checkpoint: Same as Full, but also truncates the WAL file after - /// successful checkpoint. Use this to reclaim disk space. - /// + /// + /// Truncate checkpoint: same as but truncates WAL after + /// successfully applying committed pages. Use this to reclaim disk space. + /// Truncate = 2, - /// - /// Restart checkpoint: Truncates WAL and restarts with a new WAL file. - /// Forces a fresh start. Most aggressive mode. - /// - Restart = 3 -} + /// + /// Restart checkpoint: same as and then reinitializes + /// the WAL stream for a fresh writer session. + /// + Restart = 3 +} + +/// +/// Result of a checkpoint execution. +/// +/// Requested checkpoint mode. +/// True when checkpoint logic ran; false when skipped (for passive mode contention). +/// Number of pages copied from WAL index to page file. +/// WAL size before the operation. +/// WAL size after the operation. +/// True when WAL was truncated by this operation. +/// True when WAL stream was restarted by this operation. +public readonly record struct CheckpointResult( + CheckpointMode Mode, + bool Executed, + int AppliedPages, + long WalBytesBefore, + long WalBytesAfter, + bool Truncated, + bool Restarted); diff --git a/src/CBDD.Core/Transactions/WriteAheadLog.cs b/src/CBDD.Core/Transactions/WriteAheadLog.cs index 953cb0d..4ede234 100755 --- a/src/CBDD.Core/Transactions/WriteAheadLog.cs +++ b/src/CBDD.Core/Transactions/WriteAheadLog.cs @@ -214,15 +214,70 @@ public sealed class WriteAheadLog : IDisposable } } - private void WriteAbortRecordInternal(ulong transactionId) - { - Span buffer = stackalloc byte[17]; // type(1) + txnId(8) + timestamp(8) - buffer[0] = (byte)WalRecordType.Abort; + private void WriteAbortRecordInternal(ulong transactionId) + { + Span buffer = stackalloc byte[17]; // type(1) + txnId(8) + timestamp(8) + buffer[0] = (byte)WalRecordType.Abort; BitConverter.TryWriteBytes(buffer[1..9], transactionId); BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - _walStream!.Write(buffer); - } + _walStream!.Write(buffer); + } + + + /// + /// Writes a checkpoint marker record. + /// + public void WriteCheckpointRecord() + { + _lock.Wait(); + try + { + WriteCheckpointRecordInternal(); + } + finally + { + _lock.Release(); + } + } + + /// + /// Writes a checkpoint marker record asynchronously. + /// + /// The cancellation token. + /// A task that represents the asynchronous write operation. + public async ValueTask WriteCheckpointRecordAsync(CancellationToken ct = default) + { + await _lock.WaitAsync(ct); + try + { + var buffer = System.Buffers.ArrayPool.Shared.Rent(17); + try + { + buffer[0] = (byte)WalRecordType.Checkpoint; + BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), 0UL); + BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + await _walStream!.WriteAsync(new ReadOnlyMemory(buffer, 0, 17), ct); + } + finally + { + System.Buffers.ArrayPool.Shared.Return(buffer); + } + } + finally + { + _lock.Release(); + } + } + + private void WriteCheckpointRecordInternal() + { + Span buffer = stackalloc byte[17]; // type(1) + reserved(8) + timestamp(8) + buffer[0] = (byte)WalRecordType.Checkpoint; + BitConverter.TryWriteBytes(buffer[1..9], 0UL); + BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + _walStream!.Write(buffer); + } /// @@ -381,10 +436,10 @@ public sealed class WriteAheadLog : IDisposable /// Truncates the WAL file (removes all content). /// Should only be called after successful checkpoint. /// - public void Truncate() - { - _lock.Wait(); - try + public void Truncate() + { + _lock.Wait(); + try { if (_walStream != null) { @@ -395,9 +450,31 @@ public sealed class WriteAheadLog : IDisposable } finally { - _lock.Release(); - } - } + _lock.Release(); + } + } + + /// + /// Truncates and reopens the WAL stream to start a fresh writer session. + /// + public void Restart() + { + _lock.Wait(); + try + { + _walStream?.Dispose(); + _walStream = new FileStream( + _walPath, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 64 * 1024); + } + finally + { + _lock.Release(); + } + } /// @@ -437,9 +514,10 @@ public sealed class WriteAheadLog : IDisposable switch (type) { - case WalRecordType.Begin: - case WalRecordType.Commit: - case WalRecordType.Abort: + case WalRecordType.Begin: + case WalRecordType.Commit: + case WalRecordType.Abort: + case WalRecordType.Checkpoint: // Read common fields (txnId + timestamp = 16 bytes) var bytesRead = _walStream.Read(headerBuf); if (bytesRead < 16) diff --git a/src/CBDD.SourceGenerators/EntityAnalyzer.cs b/src/CBDD.SourceGenerators/Analysis/EntityAnalyzer.cs similarity index 100% rename from src/CBDD.SourceGenerators/EntityAnalyzer.cs rename to src/CBDD.SourceGenerators/Analysis/EntityAnalyzer.cs diff --git a/src/CBDD.SourceGenerators/CodeGenerator.cs b/src/CBDD.SourceGenerators/Generators/CodeGenerator.cs similarity index 100% rename from src/CBDD.SourceGenerators/CodeGenerator.cs rename to src/CBDD.SourceGenerators/Generators/CodeGenerator.cs diff --git a/src/CBDD.SourceGenerators/MapperGenerator.cs b/src/CBDD.SourceGenerators/Generators/MapperGenerator.cs similarity index 100% rename from src/CBDD.SourceGenerators/MapperGenerator.cs rename to src/CBDD.SourceGenerators/Generators/MapperGenerator.cs diff --git a/src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj b/src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj index b0f6baf..4214045 100755 --- a/src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj +++ b/src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj @@ -1,14 +1,14 @@ - - netstandard2.0 - ZB.MOM.WW.CBDD.SourceGenerators - ZB.MOM.WW.CBDD.SourceGenerators - latest - enable + + net10.0 + ZB.MOM.WW.CBDD.SourceGenerators + ZB.MOM.WW.CBDD.SourceGenerators + latest + enable true true - ZB.MOM.WW.CBDD.SourceGenerators + ZB.MOM.WW.CBDD.SourceGenerators 1.3.1 CBDD Team Source Generators for CBDD High-Performance BSON Database Engine @@ -29,12 +29,10 @@ - - - + diff --git a/src/CBDD.SourceGenerators/_._ b/src/CBDD.SourceGenerators/_._ deleted file mode 100755 index e69de29..0000000 diff --git a/tests/CBDD.Tests.Benchmark/CompactionBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Compaction/CompactionBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/CompactionBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Compaction/CompactionBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/CompressionBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Compression/CompressionBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/CompressionBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Compression/CompressionBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/Person.cs b/tests/CBDD.Tests.Benchmark/Fixtures/Person.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/Person.cs rename to tests/CBDD.Tests.Benchmark/Fixtures/Person.cs diff --git a/tests/CBDD.Tests.Benchmark/PersonMapper.cs b/tests/CBDD.Tests.Benchmark/Fixtures/PersonMapper.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/PersonMapper.cs rename to tests/CBDD.Tests.Benchmark/Fixtures/PersonMapper.cs diff --git a/tests/CBDD.Tests.Benchmark/BenchmarkTransactionHolder.cs b/tests/CBDD.Tests.Benchmark/Infrastructure/BenchmarkTransactionHolder.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/BenchmarkTransactionHolder.cs rename to tests/CBDD.Tests.Benchmark/Infrastructure/BenchmarkTransactionHolder.cs diff --git a/tests/CBDD.Tests.Benchmark/Logging.cs b/tests/CBDD.Tests.Benchmark/Infrastructure/Logging.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/Logging.cs rename to tests/CBDD.Tests.Benchmark/Infrastructure/Logging.cs diff --git a/tests/CBDD.Tests.Benchmark/Program.cs b/tests/CBDD.Tests.Benchmark/Infrastructure/Program.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/Program.cs rename to tests/CBDD.Tests.Benchmark/Infrastructure/Program.cs diff --git a/tests/CBDD.Tests.Benchmark/InsertBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Insert/InsertBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/InsertBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Insert/InsertBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/ReadBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Read/ReadBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/ReadBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Read/ReadBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/SerializationBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Serialization/SerializationBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/SerializationBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Serialization/SerializationBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/DatabaseSizeBenchmark.cs b/tests/CBDD.Tests.Benchmark/Storage/DatabaseSizeBenchmark.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/DatabaseSizeBenchmark.cs rename to tests/CBDD.Tests.Benchmark/Storage/DatabaseSizeBenchmark.cs diff --git a/tests/CBDD.Tests.Benchmark/ManualBenchmark.cs b/tests/CBDD.Tests.Benchmark/Workloads/ManualBenchmark.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/ManualBenchmark.cs rename to tests/CBDD.Tests.Benchmark/Workloads/ManualBenchmark.cs diff --git a/tests/CBDD.Tests.Benchmark/MixedWorkloadBenchmarks.cs b/tests/CBDD.Tests.Benchmark/Workloads/MixedWorkloadBenchmarks.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/MixedWorkloadBenchmarks.cs rename to tests/CBDD.Tests.Benchmark/Workloads/MixedWorkloadBenchmarks.cs diff --git a/tests/CBDD.Tests.Benchmark/PerformanceGateSmoke.cs b/tests/CBDD.Tests.Benchmark/Workloads/PerformanceGateSmoke.cs similarity index 100% rename from tests/CBDD.Tests.Benchmark/PerformanceGateSmoke.cs rename to tests/CBDD.Tests.Benchmark/Workloads/PerformanceGateSmoke.cs diff --git a/tests/CBDD.Tests/ArchitectureFitnessTests.cs b/tests/CBDD.Tests/Architecture/ArchitectureFitnessTests.cs similarity index 100% rename from tests/CBDD.Tests/ArchitectureFitnessTests.cs rename to tests/CBDD.Tests/Architecture/ArchitectureFitnessTests.cs diff --git a/tests/CBDD.Tests/BsonDocumentAndBufferWriterTests.cs b/tests/CBDD.Tests/Bson/BsonDocumentAndBufferWriterTests.cs similarity index 100% rename from tests/CBDD.Tests/BsonDocumentAndBufferWriterTests.cs rename to tests/CBDD.Tests/Bson/BsonDocumentAndBufferWriterTests.cs diff --git a/tests/CBDD.Tests/BsonSchemaTests.cs b/tests/CBDD.Tests/Bson/BsonSchemaTests.cs similarity index 100% rename from tests/CBDD.Tests/BsonSchemaTests.cs rename to tests/CBDD.Tests/Bson/BsonSchemaTests.cs diff --git a/tests/CBDD.Tests/BsonSpanReaderWriterTests.cs b/tests/CBDD.Tests/Bson/BsonSpanReaderWriterTests.cs similarity index 100% rename from tests/CBDD.Tests/BsonSpanReaderWriterTests.cs rename to tests/CBDD.Tests/Bson/BsonSpanReaderWriterTests.cs diff --git a/tests/CBDD.Tests/CdcScalabilityTests.cs b/tests/CBDD.Tests/Cdc/CdcScalabilityTests.cs similarity index 100% rename from tests/CBDD.Tests/CdcScalabilityTests.cs rename to tests/CBDD.Tests/Cdc/CdcScalabilityTests.cs diff --git a/tests/CBDD.Tests/CdcTests.cs b/tests/CBDD.Tests/Cdc/CdcTests.cs similarity index 100% rename from tests/CBDD.Tests/CdcTests.cs rename to tests/CBDD.Tests/Cdc/CdcTests.cs diff --git a/tests/CBDD.Tests/AsyncTests.cs b/tests/CBDD.Tests/Collections/AsyncTests.cs similarity index 100% rename from tests/CBDD.Tests/AsyncTests.cs rename to tests/CBDD.Tests/Collections/AsyncTests.cs diff --git a/tests/CBDD.Tests/BulkOperationsTests.cs b/tests/CBDD.Tests/Collections/BulkOperationsTests.cs similarity index 100% rename from tests/CBDD.Tests/BulkOperationsTests.cs rename to tests/CBDD.Tests/Collections/BulkOperationsTests.cs diff --git a/tests/CBDD.Tests/DocumentCollectionDeleteTests.cs b/tests/CBDD.Tests/Collections/DocumentCollectionDeleteTests.cs similarity index 100% rename from tests/CBDD.Tests/DocumentCollectionDeleteTests.cs rename to tests/CBDD.Tests/Collections/DocumentCollectionDeleteTests.cs diff --git a/tests/CBDD.Tests/DocumentCollectionIndexApiTests.cs b/tests/CBDD.Tests/Collections/DocumentCollectionIndexApiTests.cs similarity index 100% rename from tests/CBDD.Tests/DocumentCollectionIndexApiTests.cs rename to tests/CBDD.Tests/Collections/DocumentCollectionIndexApiTests.cs diff --git a/tests/CBDD.Tests/DocumentCollectionTests.cs b/tests/CBDD.Tests/Collections/DocumentCollectionTests.cs similarity index 100% rename from tests/CBDD.Tests/DocumentCollectionTests.cs rename to tests/CBDD.Tests/Collections/DocumentCollectionTests.cs diff --git a/tests/CBDD.Tests/InsertBulkTests.cs b/tests/CBDD.Tests/Collections/InsertBulkTests.cs similarity index 100% rename from tests/CBDD.Tests/InsertBulkTests.cs rename to tests/CBDD.Tests/Collections/InsertBulkTests.cs diff --git a/tests/CBDD.Tests/SetMethodTests.cs b/tests/CBDD.Tests/Collections/SetMethodTests.cs similarity index 100% rename from tests/CBDD.Tests/SetMethodTests.cs rename to tests/CBDD.Tests/Collections/SetMethodTests.cs diff --git a/tests/CBDD.Tests/CompactionCrashRecoveryTests.cs b/tests/CBDD.Tests/Compaction/CompactionCrashRecoveryTests.cs similarity index 100% rename from tests/CBDD.Tests/CompactionCrashRecoveryTests.cs rename to tests/CBDD.Tests/Compaction/CompactionCrashRecoveryTests.cs diff --git a/tests/CBDD.Tests/CompactionOfflineTests.cs b/tests/CBDD.Tests/Compaction/CompactionOfflineTests.cs similarity index 100% rename from tests/CBDD.Tests/CompactionOfflineTests.cs rename to tests/CBDD.Tests/Compaction/CompactionOfflineTests.cs diff --git a/tests/CBDD.Tests/CompactionOnlineConcurrencyTests.cs b/tests/CBDD.Tests/Compaction/CompactionOnlineConcurrencyTests.cs similarity index 100% rename from tests/CBDD.Tests/CompactionOnlineConcurrencyTests.cs rename to tests/CBDD.Tests/Compaction/CompactionOnlineConcurrencyTests.cs diff --git a/tests/CBDD.Tests/CompactionWalCoordinationTests.cs b/tests/CBDD.Tests/Compaction/CompactionWalCoordinationTests.cs similarity index 100% rename from tests/CBDD.Tests/CompactionWalCoordinationTests.cs rename to tests/CBDD.Tests/Compaction/CompactionWalCoordinationTests.cs diff --git a/tests/CBDD.Tests/CompressionCompatibilityTests.cs b/tests/CBDD.Tests/Compression/CompressionCompatibilityTests.cs similarity index 100% rename from tests/CBDD.Tests/CompressionCompatibilityTests.cs rename to tests/CBDD.Tests/Compression/CompressionCompatibilityTests.cs diff --git a/tests/CBDD.Tests/CompressionCorruptionTests.cs b/tests/CBDD.Tests/Compression/CompressionCorruptionTests.cs similarity index 100% rename from tests/CBDD.Tests/CompressionCorruptionTests.cs rename to tests/CBDD.Tests/Compression/CompressionCorruptionTests.cs diff --git a/tests/CBDD.Tests/CompressionInsertReadTests.cs b/tests/CBDD.Tests/Compression/CompressionInsertReadTests.cs similarity index 100% rename from tests/CBDD.Tests/CompressionInsertReadTests.cs rename to tests/CBDD.Tests/Compression/CompressionInsertReadTests.cs diff --git a/tests/CBDD.Tests/CompressionOverflowTests.cs b/tests/CBDD.Tests/Compression/CompressionOverflowTests.cs similarity index 100% rename from tests/CBDD.Tests/CompressionOverflowTests.cs rename to tests/CBDD.Tests/Compression/CompressionOverflowTests.cs diff --git a/tests/CBDD.Tests/AutoInitTests.cs b/tests/CBDD.Tests/Context/AutoInitTests.cs similarity index 100% rename from tests/CBDD.Tests/AutoInitTests.cs rename to tests/CBDD.Tests/Context/AutoInitTests.cs diff --git a/tests/CBDD.Tests/DbContextInheritanceTests.cs b/tests/CBDD.Tests/Context/DbContextInheritanceTests.cs similarity index 100% rename from tests/CBDD.Tests/DbContextInheritanceTests.cs rename to tests/CBDD.Tests/Context/DbContextInheritanceTests.cs diff --git a/tests/CBDD.Tests/DbContextTests.cs b/tests/CBDD.Tests/Context/DbContextTests.cs similarity index 100% rename from tests/CBDD.Tests/DbContextTests.cs rename to tests/CBDD.Tests/Context/DbContextTests.cs diff --git a/tests/CBDD.Tests/SourceGeneratorFeaturesTests.cs b/tests/CBDD.Tests/Context/SourceGeneratorFeaturesTests.cs similarity index 100% rename from tests/CBDD.Tests/SourceGeneratorFeaturesTests.cs rename to tests/CBDD.Tests/Context/SourceGeneratorFeaturesTests.cs diff --git a/tests/CBDD.Tests/TestDbContext.cs b/tests/CBDD.Tests/Context/TestDbContext.cs similarity index 95% rename from tests/CBDD.Tests/TestDbContext.cs rename to tests/CBDD.Tests/Context/TestDbContext.cs index 34408a1..5435eff 100755 --- a/tests/CBDD.Tests/TestDbContext.cs +++ b/tests/CBDD.Tests/Context/TestDbContext.cs @@ -2,9 +2,10 @@ using ZB.MOM.WW.CBDD.Bson; using ZB.MOM.WW.CBDD.Core; using ZB.MOM.WW.CBDD.Core.Collections; using ZB.MOM.WW.CBDD.Core.Compression; -using ZB.MOM.WW.CBDD.Core.Indexing; -using ZB.MOM.WW.CBDD.Core.Metadata; -using ZB.MOM.WW.CBDD.Core.Storage; +using ZB.MOM.WW.CBDD.Core.Indexing; +using ZB.MOM.WW.CBDD.Core.Metadata; +using ZB.MOM.WW.CBDD.Core.Storage; +using ZB.MOM.WW.CBDD.Core.Transactions; namespace ZB.MOM.WW.CBDD.Shared; @@ -235,16 +236,25 @@ public partial class TestDbContext : DocumentDbContext modelBuilder.Entity().ToCollection("temporal_entities").HasKey(e => e.Id); } - /// - /// Executes ForceCheckpoint. - /// - public void ForceCheckpoint() - { - Engine.Checkpoint(); - } - - /// - /// Gets or sets the Storage. - /// + /// + /// Executes ForceCheckpoint. + /// + public void ForceCheckpoint() + { + Engine.Checkpoint(); + } + + /// + /// Executes ForceCheckpoint with the requested checkpoint mode. + /// + /// Checkpoint mode to execute. + public CheckpointResult ForceCheckpoint(CheckpointMode mode) + { + return Engine.Checkpoint(mode); + } + + /// + /// Gets or sets the Storage. + /// public StorageEngine Storage => Engine; } diff --git a/tests/CBDD.Tests/TestExtendedDbContext.cs b/tests/CBDD.Tests/Context/TestExtendedDbContext.cs similarity index 100% rename from tests/CBDD.Tests/TestExtendedDbContext.cs rename to tests/CBDD.Tests/Context/TestExtendedDbContext.cs diff --git a/tests/CBDD.Tests/MockEntities.cs b/tests/CBDD.Tests/Fixtures/MockEntities.cs similarity index 100% rename from tests/CBDD.Tests/MockEntities.cs rename to tests/CBDD.Tests/Fixtures/MockEntities.cs diff --git a/tests/CBDD.Tests/BTreeDeleteUnderflowTests.cs b/tests/CBDD.Tests/Indexing/BTreeDeleteUnderflowTests.cs similarity index 100% rename from tests/CBDD.Tests/BTreeDeleteUnderflowTests.cs rename to tests/CBDD.Tests/Indexing/BTreeDeleteUnderflowTests.cs diff --git a/tests/CBDD.Tests/CollectionIndexManagerAndDefinitionTests.cs b/tests/CBDD.Tests/Indexing/CollectionIndexManagerAndDefinitionTests.cs similarity index 100% rename from tests/CBDD.Tests/CollectionIndexManagerAndDefinitionTests.cs rename to tests/CBDD.Tests/Indexing/CollectionIndexManagerAndDefinitionTests.cs diff --git a/tests/CBDD.Tests/CursorTests.cs b/tests/CBDD.Tests/Indexing/CursorTests.cs similarity index 100% rename from tests/CBDD.Tests/CursorTests.cs rename to tests/CBDD.Tests/Indexing/CursorTests.cs diff --git a/tests/CBDD.Tests/GeospatialStressTests.cs b/tests/CBDD.Tests/Indexing/GeospatialStressTests.cs similarity index 100% rename from tests/CBDD.Tests/GeospatialStressTests.cs rename to tests/CBDD.Tests/Indexing/GeospatialStressTests.cs diff --git a/tests/CBDD.Tests/GeospatialTests.cs b/tests/CBDD.Tests/Indexing/GeospatialTests.cs similarity index 100% rename from tests/CBDD.Tests/GeospatialTests.cs rename to tests/CBDD.Tests/Indexing/GeospatialTests.cs diff --git a/tests/CBDD.Tests/HashIndexTests.cs b/tests/CBDD.Tests/Indexing/HashIndexTests.cs similarity index 100% rename from tests/CBDD.Tests/HashIndexTests.cs rename to tests/CBDD.Tests/Indexing/HashIndexTests.cs diff --git a/tests/CBDD.Tests/IndexDirectionTests.cs b/tests/CBDD.Tests/Indexing/IndexDirectionTests.cs similarity index 100% rename from tests/CBDD.Tests/IndexDirectionTests.cs rename to tests/CBDD.Tests/Indexing/IndexDirectionTests.cs diff --git a/tests/CBDD.Tests/IndexOptimizationTests.cs b/tests/CBDD.Tests/Indexing/IndexOptimizationTests.cs similarity index 100% rename from tests/CBDD.Tests/IndexOptimizationTests.cs rename to tests/CBDD.Tests/Indexing/IndexOptimizationTests.cs diff --git a/tests/CBDD.Tests/PrimaryKeyTests.cs b/tests/CBDD.Tests/Indexing/PrimaryKeyTests.cs similarity index 100% rename from tests/CBDD.Tests/PrimaryKeyTests.cs rename to tests/CBDD.Tests/Indexing/PrimaryKeyTests.cs diff --git a/tests/CBDD.Tests/VectorMathTests.cs b/tests/CBDD.Tests/Indexing/VectorMathTests.cs similarity index 100% rename from tests/CBDD.Tests/VectorMathTests.cs rename to tests/CBDD.Tests/Indexing/VectorMathTests.cs diff --git a/tests/CBDD.Tests/VectorSearchTests.cs b/tests/CBDD.Tests/Indexing/VectorSearchTests.cs similarity index 100% rename from tests/CBDD.Tests/VectorSearchTests.cs rename to tests/CBDD.Tests/Indexing/VectorSearchTests.cs diff --git a/tests/CBDD.Tests/WalIndexTests.cs b/tests/CBDD.Tests/Indexing/WalIndexTests.cs similarity index 100% rename from tests/CBDD.Tests/WalIndexTests.cs rename to tests/CBDD.Tests/Indexing/WalIndexTests.cs diff --git a/tests/CBDD.Tests/AdvancedQueryTests.cs b/tests/CBDD.Tests/Query/AdvancedQueryTests.cs similarity index 100% rename from tests/CBDD.Tests/AdvancedQueryTests.cs rename to tests/CBDD.Tests/Query/AdvancedQueryTests.cs diff --git a/tests/CBDD.Tests/LinqTests.cs b/tests/CBDD.Tests/Query/LinqTests.cs similarity index 100% rename from tests/CBDD.Tests/LinqTests.cs rename to tests/CBDD.Tests/Query/LinqTests.cs diff --git a/tests/CBDD.Tests/QueryPrimitivesTests.cs b/tests/CBDD.Tests/Query/QueryPrimitivesTests.cs similarity index 100% rename from tests/CBDD.Tests/QueryPrimitivesTests.cs rename to tests/CBDD.Tests/Query/QueryPrimitivesTests.cs diff --git a/tests/CBDD.Tests/ScanTests.cs b/tests/CBDD.Tests/Query/ScanTests.cs similarity index 100% rename from tests/CBDD.Tests/ScanTests.cs rename to tests/CBDD.Tests/Query/ScanTests.cs diff --git a/tests/CBDD.Tests/AttributeTests.cs b/tests/CBDD.Tests/Schema/AttributeTests.cs similarity index 100% rename from tests/CBDD.Tests/AttributeTests.cs rename to tests/CBDD.Tests/Schema/AttributeTests.cs diff --git a/tests/CBDD.Tests/CircularReferenceTests.cs b/tests/CBDD.Tests/Schema/CircularReferenceTests.cs similarity index 100% rename from tests/CBDD.Tests/CircularReferenceTests.cs rename to tests/CBDD.Tests/Schema/CircularReferenceTests.cs diff --git a/tests/CBDD.Tests/NullableStringIdTests.cs b/tests/CBDD.Tests/Schema/NullableStringIdTests.cs similarity index 100% rename from tests/CBDD.Tests/NullableStringIdTests.cs rename to tests/CBDD.Tests/Schema/NullableStringIdTests.cs diff --git a/tests/CBDD.Tests/SchemaPersistenceTests.cs b/tests/CBDD.Tests/Schema/SchemaPersistenceTests.cs similarity index 100% rename from tests/CBDD.Tests/SchemaPersistenceTests.cs rename to tests/CBDD.Tests/Schema/SchemaPersistenceTests.cs diff --git a/tests/CBDD.Tests/SchemaTests.cs b/tests/CBDD.Tests/Schema/SchemaTests.cs similarity index 100% rename from tests/CBDD.Tests/SchemaTests.cs rename to tests/CBDD.Tests/Schema/SchemaTests.cs diff --git a/tests/CBDD.Tests/TemporalTypesTests.cs b/tests/CBDD.Tests/Schema/TemporalTypesTests.cs similarity index 100% rename from tests/CBDD.Tests/TemporalTypesTests.cs rename to tests/CBDD.Tests/Schema/TemporalTypesTests.cs diff --git a/tests/CBDD.Tests/VisibilityTests.cs b/tests/CBDD.Tests/Schema/VisibilityTests.cs similarity index 100% rename from tests/CBDD.Tests/VisibilityTests.cs rename to tests/CBDD.Tests/Schema/VisibilityTests.cs diff --git a/tests/CBDD.Tests/Storage/CheckpointModeTests.cs b/tests/CBDD.Tests/Storage/CheckpointModeTests.cs new file mode 100644 index 0000000..91c408e --- /dev/null +++ b/tests/CBDD.Tests/Storage/CheckpointModeTests.cs @@ -0,0 +1,228 @@ +using System.Reflection; +using ZB.MOM.WW.CBDD.Bson; +using ZB.MOM.WW.CBDD.Core.Storage; +using ZB.MOM.WW.CBDD.Core.Transactions; +using ZB.MOM.WW.CBDD.Shared; + +namespace ZB.MOM.WW.CBDD.Tests; + +public class CheckpointModeTests +{ + /// + /// Verifies default checkpoint mode truncates WAL. + /// + [Fact] + public void Checkpoint_Default_ShouldUseTruncate() + { + var dbPath = NewDbPath(); + try + { + using var db = new TestDbContext(dbPath); + db.Users.Insert(new User { Name = "checkpoint-default", Age = 42 }); + db.SaveChanges(); + + db.Storage.GetWalSize().ShouldBeGreaterThan(0); + var result = db.Checkpoint(); + + result.Mode.ShouldBe(CheckpointMode.Truncate); + result.Executed.ShouldBeTrue(); + result.Truncated.ShouldBeTrue(); + db.Storage.GetWalSize().ShouldBe(0); + } + finally + { + CleanupFiles(dbPath); + } + } + + /// + /// Verifies passive mode skips when checkpoint lock is contended. + /// + [Fact] + public void Checkpoint_Passive_ShouldSkip_WhenLockIsContended() + { + var dbPath = NewDbPath(); + try + { + using var storage = new StorageEngine(dbPath, PageFileConfig.Default); + var gate = GetCommitGate(storage); + gate.Wait(TestContext.Current.CancellationToken); + try + { + var result = storage.Checkpoint(CheckpointMode.Passive); + result.Mode.ShouldBe(CheckpointMode.Passive); + result.Executed.ShouldBeFalse(); + result.Truncated.ShouldBeFalse(); + result.Restarted.ShouldBeFalse(); + } + finally + { + gate.Release(); + } + } + finally + { + CleanupFiles(dbPath); + } + } + + /// + /// Verifies full checkpoint applies data and appends a checkpoint marker without truncating WAL. + /// + [Fact] + public void Checkpoint_Full_ShouldAppendMarker_AndPreserveWal() + { + var dbPath = NewDbPath(); + var walPath = Path.ChangeExtension(dbPath, ".wal"); + + try + { + using (var db = new TestDbContext(dbPath)) + { + db.Users.Insert(new User { Name = "checkpoint-full", Age = 50 }); + db.SaveChanges(); + + var walBefore = db.Storage.GetWalSize(); + walBefore.ShouldBeGreaterThan(0); + + var result = db.Checkpoint(CheckpointMode.Full); + result.Mode.ShouldBe(CheckpointMode.Full); + result.Executed.ShouldBeTrue(); + result.Truncated.ShouldBeFalse(); + result.WalBytesAfter.ShouldBeGreaterThan(0); + db.Storage.GetWalSize().ShouldBeGreaterThan(0); + } + + using var wal = new WriteAheadLog(walPath); + wal.ReadAll().Any(r => r.Type == WalRecordType.Checkpoint).ShouldBeTrue(); + } + finally + { + CleanupFiles(dbPath); + } + } + + /// + /// Verifies restart checkpoint clears WAL and allows subsequent writes. + /// + [Fact] + public void Checkpoint_Restart_ShouldResetWal_AndAcceptNewWrites() + { + var dbPath = NewDbPath(); + try + { + using var db = new TestDbContext(dbPath); + db.Users.Insert(new User { Name = "restart-before", Age = 30 }); + db.SaveChanges(); + + db.Storage.GetWalSize().ShouldBeGreaterThan(0); + var result = db.Checkpoint(CheckpointMode.Restart); + result.Mode.ShouldBe(CheckpointMode.Restart); + result.Executed.ShouldBeTrue(); + result.Truncated.ShouldBeTrue(); + result.Restarted.ShouldBeTrue(); + db.Storage.GetWalSize().ShouldBe(0); + + db.Users.Insert(new User { Name = "restart-after", Age = 31 }); + db.SaveChanges(); + db.Storage.GetWalSize().ShouldBeGreaterThan(0); + } + finally + { + CleanupFiles(dbPath); + } + } + + /// + /// Verifies recovery remains deterministic after a full checkpoint boundary. + /// + [Fact] + public void Recover_AfterFullCheckpoint_ShouldApplyLatestCommitDeterministically() + { + var dbPath = NewDbPath(); + try + { + uint pageId; + + using (var storage = new StorageEngine(dbPath, PageFileConfig.Default)) + { + pageId = storage.AllocatePage(); + + using (var tx1 = storage.BeginTransaction()) + { + var first = new byte[storage.PageSize]; + first[0] = 1; + storage.WritePage(pageId, tx1.TransactionId, first); + tx1.Commit(); + } + + storage.Checkpoint(CheckpointMode.Full); + + using (var tx2 = storage.BeginTransaction()) + { + var second = new byte[storage.PageSize]; + second[0] = 2; + storage.WritePage(pageId, tx2.TransactionId, second); + tx2.Commit(); + } + } + + using (var recovered = new StorageEngine(dbPath, PageFileConfig.Default)) + { + var buffer = new byte[recovered.PageSize]; + recovered.ReadPage(pageId, 0, buffer); + buffer[0].ShouldBe((byte)2); + recovered.GetWalSize().ShouldBe(0); + } + } + finally + { + CleanupFiles(dbPath); + } + } + + /// + /// Verifies asynchronous mode-based checkpoints return expected result metadata. + /// + [Fact] + public async Task CheckpointAsync_Full_ShouldReturnResult() + { + var dbPath = NewDbPath(); + try + { + using var db = new TestDbContext(dbPath); + db.Users.Insert(new User { Name = "checkpoint-async", Age = 38 }); + db.SaveChanges(); + + var result = await db.CheckpointAsync(CheckpointMode.Full, TestContext.Current.CancellationToken); + result.Mode.ShouldBe(CheckpointMode.Full); + result.Executed.ShouldBeTrue(); + result.Truncated.ShouldBeFalse(); + } + finally + { + CleanupFiles(dbPath); + } + } + + private static SemaphoreSlim GetCommitGate(StorageEngine storage) + { + var field = typeof(StorageEngine).GetField("_commitLock", BindingFlags.Instance | BindingFlags.NonPublic); + field.ShouldNotBeNull(); + return (SemaphoreSlim)field!.GetValue(storage)!; + } + + private static string NewDbPath() + => Path.Combine(Path.GetTempPath(), $"checkpoint_mode_{Guid.NewGuid():N}.db"); + + private static void CleanupFiles(string dbPath) + { + if (File.Exists(dbPath)) File.Delete(dbPath); + + var walPath = Path.ChangeExtension(dbPath, ".wal"); + if (File.Exists(walPath)) File.Delete(walPath); + + var markerPath = $"{dbPath}.compact.state"; + if (File.Exists(markerPath)) File.Delete(markerPath); + } +} diff --git a/tests/CBDD.Tests/DictionaryPageTests.cs b/tests/CBDD.Tests/Storage/DictionaryPageTests.cs similarity index 100% rename from tests/CBDD.Tests/DictionaryPageTests.cs rename to tests/CBDD.Tests/Storage/DictionaryPageTests.cs diff --git a/tests/CBDD.Tests/DictionaryPersistenceTests.cs b/tests/CBDD.Tests/Storage/DictionaryPersistenceTests.cs similarity index 100% rename from tests/CBDD.Tests/DictionaryPersistenceTests.cs rename to tests/CBDD.Tests/Storage/DictionaryPersistenceTests.cs diff --git a/tests/CBDD.Tests/DocumentOverflowTests.cs b/tests/CBDD.Tests/Storage/DocumentOverflowTests.cs similarity index 100% rename from tests/CBDD.Tests/DocumentOverflowTests.cs rename to tests/CBDD.Tests/Storage/DocumentOverflowTests.cs diff --git a/tests/CBDD.Tests/MaintenanceDiagnosticsAndMigrationTests.cs b/tests/CBDD.Tests/Storage/MaintenanceDiagnosticsAndMigrationTests.cs similarity index 100% rename from tests/CBDD.Tests/MaintenanceDiagnosticsAndMigrationTests.cs rename to tests/CBDD.Tests/Storage/MaintenanceDiagnosticsAndMigrationTests.cs diff --git a/tests/CBDD.Tests/MetadataPersistenceTests.cs b/tests/CBDD.Tests/Storage/MetadataPersistenceTests.cs similarity index 100% rename from tests/CBDD.Tests/MetadataPersistenceTests.cs rename to tests/CBDD.Tests/Storage/MetadataPersistenceTests.cs diff --git a/tests/CBDD.Tests/RobustnessTests.cs b/tests/CBDD.Tests/Storage/RobustnessTests.cs similarity index 100% rename from tests/CBDD.Tests/RobustnessTests.cs rename to tests/CBDD.Tests/Storage/RobustnessTests.cs diff --git a/tests/CBDD.Tests/StorageEngineDictionaryTests.cs b/tests/CBDD.Tests/Storage/StorageEngineDictionaryTests.cs similarity index 100% rename from tests/CBDD.Tests/StorageEngineDictionaryTests.cs rename to tests/CBDD.Tests/Storage/StorageEngineDictionaryTests.cs diff --git a/tests/CBDD.Tests/StorageEngineTransactionProtocolTests.cs b/tests/CBDD.Tests/Storage/StorageEngineTransactionProtocolTests.cs similarity index 100% rename from tests/CBDD.Tests/StorageEngineTransactionProtocolTests.cs rename to tests/CBDD.Tests/Storage/StorageEngineTransactionProtocolTests.cs diff --git a/tests/CBDD.Tests/ObjectIdTests.cs b/tests/CBDD.Tests/Types/ObjectIdTests.cs similarity index 100% rename from tests/CBDD.Tests/ObjectIdTests.cs rename to tests/CBDD.Tests/Types/ObjectIdTests.cs diff --git a/tests/CBDD.Tests/ValueObjectIdTests.cs b/tests/CBDD.Tests/Types/ValueObjectIdTests.cs similarity index 100% rename from tests/CBDD.Tests/ValueObjectIdTests.cs rename to tests/CBDD.Tests/Types/ValueObjectIdTests.cs