Implement checkpoint modes with docs/tests and reorganize project file layout
This commit is contained in:
21
README.md
21
README.md
@@ -104,6 +104,27 @@ Operational procedures, diagnostics, and escalation are documented in:
|
||||
- [`docs/runbook.md`](docs/runbook.md)
|
||||
- [`docs/troubleshooting.md`](docs/troubleshooting.md)
|
||||
|
||||
## Checkpoint Modes
|
||||
|
||||
`DocumentDbContext` and `StorageEngine` support explicit checkpoint modes via `CheckpointMode`:
|
||||
|
||||
- `Passive`: non-blocking; skips if checkpoint lock is contended.
|
||||
- `Full`: applies committed WAL pages and appends a checkpoint marker, without truncating WAL.
|
||||
- `Truncate`: applies committed WAL pages and truncates WAL.
|
||||
- `Restart`: truncate + WAL writer restart.
|
||||
|
||||
Example:
|
||||
|
||||
```csharp
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
var result = db.Checkpoint(CheckpointMode.Full);
|
||||
if (!result.Executed)
|
||||
{
|
||||
// passive checkpoint can be skipped under contention
|
||||
}
|
||||
```
|
||||
|
||||
## Security And Compliance Posture
|
||||
|
||||
- CBDD relies on host and process-level access controls.
|
||||
|
||||
@@ -29,6 +29,24 @@ Non-goals:
|
||||
- `WriteAheadLog`
|
||||
- Storage engine modules under `src/CBDD.Core/Storage`
|
||||
|
||||
Checkpoint APIs:
|
||||
- `DocumentDbContext.Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)`
|
||||
- `DocumentDbContext.CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)`
|
||||
- `StorageEngine.Checkpoint(CheckpointMode mode)`
|
||||
- `StorageEngine.CheckpointAsync(CheckpointMode mode, CancellationToken ct = default)`
|
||||
|
||||
Checkpoint modes:
|
||||
- `Passive`: non-blocking best-effort checkpoint. Returns `Executed = false` when lock is contended.
|
||||
- `Full`: applies committed WAL pages and appends a WAL checkpoint marker without truncating WAL.
|
||||
- `Truncate`: applies committed WAL pages and truncates WAL (default behavior).
|
||||
- `Restart`: same as truncate, then reinitializes WAL writer session.
|
||||
|
||||
Usage guidance:
|
||||
- Use `Passive` for background/low-priority maintenance where latency matters more than immediate WAL cleanup.
|
||||
- Use `Full` when you want durable page-file sync but prefer to preserve WAL history until a later truncate.
|
||||
- Use `Truncate` for routine manual checkpoints and disk-space recovery.
|
||||
- Use `Restart` for aggressive maintenance boundaries (for example after incident remediation flows).
|
||||
|
||||
## Permissions And Data Handling
|
||||
|
||||
- Database files require host-managed filesystem access controls.
|
||||
|
||||
@@ -28,6 +28,8 @@ Database startup fails or recovery path throws WAL/storage errors.
|
||||
### Resolution
|
||||
- Pin consumers to last known-good package.
|
||||
- Apply fix and add regression coverage in recovery/transaction tests.
|
||||
- If WAL growth is the issue, run `CheckpointMode.Truncate` (or `Restart`) instead of `Full`.
|
||||
- If foreground latency is a concern, schedule `CheckpointMode.Passive` retries and use `Truncate` during maintenance windows.
|
||||
|
||||
## Query And Index Issues
|
||||
|
||||
|
||||
@@ -329,6 +329,33 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a checkpoint using the requested mode.
|
||||
/// </summary>
|
||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||
/// <returns>The checkpoint execution result.</returns>
|
||||
public CheckpointResult Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.Checkpoint(mode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a checkpoint asynchronously using the requested mode.
|
||||
/// </summary>
|
||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The checkpoint execution result.</returns>
|
||||
public Task<CheckpointResult> CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.CheckpointAsync(mode, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||
/// </summary>
|
||||
@@ -75,7 +75,7 @@ internal static class BsonExpressionEvaluator
|
||||
|
||||
if (name == propertyName)
|
||||
{
|
||||
// Found it! Read value and compare
|
||||
// Found -> read value and compare
|
||||
return Compare(ref reader, type, targetValue, op);
|
||||
}
|
||||
|
||||
|
||||
@@ -30,148 +30,241 @@ public sealed partial class StorageEngine
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint: merges WAL into PageFile.
|
||||
/// Uses in-memory WAL index for efficiency and consistency.
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Performs a checkpoint: merges WAL into PageFile.
|
||||
/// Uses in-memory WAL index for efficiency and consistency.
|
||||
/// Performs a truncate checkpoint by default.
|
||||
/// </summary>
|
||||
public void Checkpoint()
|
||||
{
|
||||
_ = Checkpoint(CheckpointMode.Truncate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint using the requested mode.
|
||||
/// </summary>
|
||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||
/// <returns>The checkpoint execution result.</returns>
|
||||
public CheckpointResult Checkpoint(CheckpointMode mode)
|
||||
{
|
||||
bool lockAcquired;
|
||||
if (mode == CheckpointMode.Passive)
|
||||
{
|
||||
lockAcquired = _commitLock.Wait(0);
|
||||
if (!lockAcquired)
|
||||
{
|
||||
var walSize = _wal.GetCurrentSize();
|
||||
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_commitLock.Wait();
|
||||
lockAcquired = true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
CheckpointInternal();
|
||||
return CheckpointInternal(mode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockAcquired)
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void CheckpointInternal()
|
||||
{
|
||||
if (_walIndex.IsEmpty)
|
||||
{
|
||||
// WAL may still contain begin/commit records for read-only transactions.
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
=> _ = CheckpointInternal(CheckpointMode.Truncate);
|
||||
|
||||
// 1. Write all committed pages from index to PageFile
|
||||
private CheckpointResult CheckpointInternal(CheckpointMode mode)
|
||||
{
|
||||
var walBytesBefore = _wal.GetCurrentSize();
|
||||
var appliedPages = 0;
|
||||
var truncated = false;
|
||||
var restarted = false;
|
||||
|
||||
// 1. Write all committed pages from index to PageFile.
|
||||
foreach (var kvp in _walIndex)
|
||||
{
|
||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
||||
appliedPages++;
|
||||
}
|
||||
|
||||
// 2. Flush PageFile to ensure durability
|
||||
// 2. Flush PageFile to ensure durability.
|
||||
if (appliedPages > 0)
|
||||
{
|
||||
_pageFile.Flush();
|
||||
}
|
||||
|
||||
// 3. Clear in-memory WAL index (now persisted)
|
||||
// 3. Clear in-memory WAL index (now persisted).
|
||||
_walIndex.Clear();
|
||||
|
||||
// 4. Truncate WAL (all changes now in PageFile)
|
||||
// 4. Apply mode-specific WAL handling.
|
||||
switch (mode)
|
||||
{
|
||||
case CheckpointMode.Passive:
|
||||
case CheckpointMode.Full:
|
||||
if (walBytesBefore > 0 || appliedPages > 0)
|
||||
{
|
||||
_wal.WriteCheckpointRecord();
|
||||
_wal.Flush();
|
||||
}
|
||||
break;
|
||||
case CheckpointMode.Truncate:
|
||||
if (walBytesBefore > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
truncated = true;
|
||||
}
|
||||
break;
|
||||
case CheckpointMode.Restart:
|
||||
_wal.Restart();
|
||||
truncated = true;
|
||||
restarted = true;
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported checkpoint mode.");
|
||||
}
|
||||
|
||||
var walBytesAfter = _wal.GetCurrentSize();
|
||||
return new CheckpointResult(mode, true, appliedPages, walBytesBefore, walBytesAfter, truncated, restarted);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint asynchronously by merging WAL pages into the page file.
|
||||
/// Performs a truncate checkpoint asynchronously by default.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous checkpoint operation.</returns>
|
||||
public async Task CheckpointAsync(CancellationToken ct = default)
|
||||
{
|
||||
_ = await CheckpointAsync(CheckpointMode.Truncate, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint asynchronously using the requested mode.
|
||||
/// </summary>
|
||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous checkpoint operation.</returns>
|
||||
public async Task<CheckpointResult> CheckpointAsync(CheckpointMode mode, CancellationToken ct = default)
|
||||
{
|
||||
bool lockAcquired;
|
||||
if (mode == CheckpointMode.Passive)
|
||||
{
|
||||
lockAcquired = await _commitLock.WaitAsync(0, ct);
|
||||
if (!lockAcquired)
|
||||
{
|
||||
var walSize = _wal.GetCurrentSize();
|
||||
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
lockAcquired = true;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (_walIndex.IsEmpty)
|
||||
{
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Write all committed pages from index to PageFile
|
||||
// PageFile writes are sync (MMF), but that's fine as per plan (ValueTask strategy for MMF)
|
||||
foreach (var kvp in _walIndex)
|
||||
{
|
||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
// 2. Flush PageFile to ensure durability
|
||||
_pageFile.Flush();
|
||||
|
||||
// 3. Clear in-memory WAL index (now persisted)
|
||||
_walIndex.Clear();
|
||||
|
||||
// 4. Truncate WAL (all changes now in PageFile)
|
||||
// WAL truncation involves file resize and flush
|
||||
// TODO: Add TruncateAsync to WAL? For now Truncate is sync.
|
||||
_wal.Truncate();
|
||||
// Checkpoint work is synchronous over MMF/page writes for now.
|
||||
return CheckpointInternal(mode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockAcquired)
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recovers from crash by replaying WAL.
|
||||
/// Applies all committed transactions to PageFile, then truncates WAL.
|
||||
/// Applies committed transactions to PageFile in deterministic WAL order, then truncates WAL.
|
||||
/// </summary>
|
||||
public void Recover()
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
// 1. Read WAL and identify committed transactions
|
||||
// 1. Read WAL and locate the latest checkpoint boundary.
|
||||
var records = _wal.ReadAll();
|
||||
var committedTxns = new HashSet<ulong>();
|
||||
var txnWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
||||
|
||||
foreach (var record in records)
|
||||
var startIndex = 0;
|
||||
for (var i = records.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (record.Type == WalRecordType.Commit)
|
||||
committedTxns.Add(record.TransactionId);
|
||||
else if (record.Type == WalRecordType.Write)
|
||||
if (records[i].Type == WalRecordType.Checkpoint)
|
||||
{
|
||||
if (!txnWrites.ContainsKey(record.TransactionId))
|
||||
txnWrites[record.TransactionId] = new List<(uint, byte[])>();
|
||||
|
||||
if (record.AfterImage != null)
|
||||
{
|
||||
txnWrites[record.TransactionId].Add((record.PageId, record.AfterImage));
|
||||
startIndex = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Replay WAL in source order with deterministic commit application.
|
||||
var pendingWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
||||
var appliedAny = false;
|
||||
|
||||
for (var i = startIndex; i < records.Count; i++)
|
||||
{
|
||||
var record = records[i];
|
||||
switch (record.Type)
|
||||
{
|
||||
case WalRecordType.Begin:
|
||||
if (!pendingWrites.ContainsKey(record.TransactionId))
|
||||
{
|
||||
pendingWrites[record.TransactionId] = new List<(uint, byte[])>();
|
||||
}
|
||||
break;
|
||||
case WalRecordType.Write:
|
||||
if (record.AfterImage == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// 2. Apply committed transactions to PageFile
|
||||
foreach (var txnId in committedTxns)
|
||||
if (!pendingWrites.TryGetValue(record.TransactionId, out var writes))
|
||||
{
|
||||
if (!txnWrites.ContainsKey(txnId))
|
||||
continue;
|
||||
writes = new List<(uint, byte[])>();
|
||||
pendingWrites[record.TransactionId] = writes;
|
||||
}
|
||||
|
||||
foreach (var (pageId, data) in txnWrites[txnId])
|
||||
writes.Add((record.PageId, record.AfterImage));
|
||||
break;
|
||||
case WalRecordType.Commit:
|
||||
if (!pendingWrites.TryGetValue(record.TransactionId, out var committedWrites))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
foreach (var (pageId, data) in committedWrites)
|
||||
{
|
||||
_pageFile.WritePage(pageId, data);
|
||||
appliedAny = true;
|
||||
}
|
||||
|
||||
pendingWrites.Remove(record.TransactionId);
|
||||
break;
|
||||
case WalRecordType.Abort:
|
||||
pendingWrites.Remove(record.TransactionId);
|
||||
break;
|
||||
case WalRecordType.Checkpoint:
|
||||
pendingWrites.Clear();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Flush PageFile to ensure durability
|
||||
// 3. Flush PageFile to ensure durability.
|
||||
if (appliedAny)
|
||||
{
|
||||
_pageFile.Flush();
|
||||
}
|
||||
|
||||
// 4. Clear in-memory WAL index (redundant since we just recovered)
|
||||
// 4. Clear in-memory WAL index (redundant since we just recovered).
|
||||
_walIndex.Clear();
|
||||
|
||||
// 5. Truncate WAL (all changes now in PageFile)
|
||||
// 5. Truncate WAL (all changes now in PageFile).
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
|
||||
@@ -7,26 +7,46 @@ namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
public enum CheckpointMode
|
||||
{
|
||||
/// <summary>
|
||||
/// Passive checkpoint: Non-blocking, best-effort transfer from WAL to database.
|
||||
/// Does not wait for readers or writers. May not checkpoint all frames.
|
||||
/// Passive checkpoint: non-blocking, best-effort transfer from WAL to database.
|
||||
/// If the checkpoint lock is busy, the operation is skipped.
|
||||
/// WAL content is preserved and a checkpoint marker is appended when work is applied.
|
||||
/// </summary>
|
||||
Passive = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Full checkpoint: Waits for concurrent readers/writers, then checkpoints all
|
||||
/// committed transactions from WAL to database. Blocks until complete.
|
||||
/// Full checkpoint: waits for the checkpoint lock, transfers committed pages to
|
||||
/// the page file, and preserves WAL content by appending a checkpoint marker.
|
||||
/// </summary>
|
||||
Full = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Truncate checkpoint: Same as Full, but also truncates the WAL file after
|
||||
/// successful checkpoint. Use this to reclaim disk space.
|
||||
/// Truncate checkpoint: same as <see cref="Full"/> but truncates WAL after
|
||||
/// successfully applying committed pages. Use this to reclaim disk space.
|
||||
/// </summary>
|
||||
Truncate = 2,
|
||||
|
||||
/// <summary>
|
||||
/// Restart checkpoint: Truncates WAL and restarts with a new WAL file.
|
||||
/// Forces a fresh start. Most aggressive mode.
|
||||
/// Restart checkpoint: same as <see cref="Truncate"/> and then reinitializes
|
||||
/// the WAL stream for a fresh writer session.
|
||||
/// </summary>
|
||||
Restart = 3
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a checkpoint execution.
|
||||
/// </summary>
|
||||
/// <param name="Mode">Requested checkpoint mode.</param>
|
||||
/// <param name="Executed">True when checkpoint logic ran; false when skipped (for passive mode contention).</param>
|
||||
/// <param name="AppliedPages">Number of pages copied from WAL index to page file.</param>
|
||||
/// <param name="WalBytesBefore">WAL size before the operation.</param>
|
||||
/// <param name="WalBytesAfter">WAL size after the operation.</param>
|
||||
/// <param name="Truncated">True when WAL was truncated by this operation.</param>
|
||||
/// <param name="Restarted">True when WAL stream was restarted by this operation.</param>
|
||||
public readonly record struct CheckpointResult(
|
||||
CheckpointMode Mode,
|
||||
bool Executed,
|
||||
int AppliedPages,
|
||||
long WalBytesBefore,
|
||||
long WalBytesAfter,
|
||||
bool Truncated,
|
||||
bool Restarted);
|
||||
|
||||
@@ -225,6 +225,61 @@ public sealed class WriteAheadLog : IDisposable
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes a checkpoint marker record.
|
||||
/// </summary>
|
||||
public void WriteCheckpointRecord()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
WriteCheckpointRecordInternal();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a checkpoint marker record asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||
public async ValueTask WriteCheckpointRecordAsync(CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(17);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Checkpoint;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), 0UL);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, 17), ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteCheckpointRecordInternal()
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[17]; // type(1) + reserved(8) + timestamp(8)
|
||||
buffer[0] = (byte)WalRecordType.Checkpoint;
|
||||
BitConverter.TryWriteBytes(buffer[1..9], 0UL);
|
||||
BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
_walStream!.Write(buffer);
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes a data modification record
|
||||
/// </summary>
|
||||
@@ -399,6 +454,28 @@ public sealed class WriteAheadLog : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Truncates and reopens the WAL stream to start a fresh writer session.
|
||||
/// </summary>
|
||||
public void Restart()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
_walStream?.Dispose();
|
||||
_walStream = new FileStream(
|
||||
_walPath,
|
||||
FileMode.Create,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 64 * 1024);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Reads all WAL records (for recovery)
|
||||
@@ -440,6 +517,7 @@ public sealed class WriteAheadLog : IDisposable
|
||||
case WalRecordType.Begin:
|
||||
case WalRecordType.Commit:
|
||||
case WalRecordType.Abort:
|
||||
case WalRecordType.Checkpoint:
|
||||
// Read common fields (txnId + timestamp = 16 bytes)
|
||||
var bytesRead = _walStream.Read(headerBuf);
|
||||
if (bytesRead < 16)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
||||
<LangVersion>latest</LangVersion>
|
||||
@@ -29,8 +29,6 @@
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
|
||||
|
||||
<None Include="_._" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -5,6 +5,7 @@ using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using ZB.MOM.WW.CBDD.Core.Metadata;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Shared;
|
||||
|
||||
@@ -243,6 +244,15 @@ public partial class TestDbContext : DocumentDbContext
|
||||
Engine.Checkpoint();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes ForceCheckpoint with the requested checkpoint mode.
|
||||
/// </summary>
|
||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||
public CheckpointResult ForceCheckpoint(CheckpointMode mode)
|
||||
{
|
||||
return Engine.Checkpoint(mode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the Storage.
|
||||
/// </summary>
|
||||
228
tests/CBDD.Tests/Storage/CheckpointModeTests.cs
Normal file
228
tests/CBDD.Tests/Storage/CheckpointModeTests.cs
Normal file
@@ -0,0 +1,228 @@
|
||||
using System.Reflection;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using ZB.MOM.WW.CBDD.Shared;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Tests;
|
||||
|
||||
public class CheckpointModeTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies default checkpoint mode truncates WAL.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Checkpoint_Default_ShouldUseTruncate()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
db.Users.Insert(new User { Name = "checkpoint-default", Age = 42 });
|
||||
db.SaveChanges();
|
||||
|
||||
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||
var result = db.Checkpoint();
|
||||
|
||||
result.Mode.ShouldBe(CheckpointMode.Truncate);
|
||||
result.Executed.ShouldBeTrue();
|
||||
result.Truncated.ShouldBeTrue();
|
||||
db.Storage.GetWalSize().ShouldBe(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies passive mode skips when checkpoint lock is contended.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Checkpoint_Passive_ShouldSkip_WhenLockIsContended()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
try
|
||||
{
|
||||
using var storage = new StorageEngine(dbPath, PageFileConfig.Default);
|
||||
var gate = GetCommitGate(storage);
|
||||
gate.Wait(TestContext.Current.CancellationToken);
|
||||
try
|
||||
{
|
||||
var result = storage.Checkpoint(CheckpointMode.Passive);
|
||||
result.Mode.ShouldBe(CheckpointMode.Passive);
|
||||
result.Executed.ShouldBeFalse();
|
||||
result.Truncated.ShouldBeFalse();
|
||||
result.Restarted.ShouldBeFalse();
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.Release();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies full checkpoint applies data and appends a checkpoint marker without truncating WAL.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Checkpoint_Full_ShouldAppendMarker_AndPreserveWal()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||
|
||||
try
|
||||
{
|
||||
using (var db = new TestDbContext(dbPath))
|
||||
{
|
||||
db.Users.Insert(new User { Name = "checkpoint-full", Age = 50 });
|
||||
db.SaveChanges();
|
||||
|
||||
var walBefore = db.Storage.GetWalSize();
|
||||
walBefore.ShouldBeGreaterThan(0);
|
||||
|
||||
var result = db.Checkpoint(CheckpointMode.Full);
|
||||
result.Mode.ShouldBe(CheckpointMode.Full);
|
||||
result.Executed.ShouldBeTrue();
|
||||
result.Truncated.ShouldBeFalse();
|
||||
result.WalBytesAfter.ShouldBeGreaterThan(0);
|
||||
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||
}
|
||||
|
||||
using var wal = new WriteAheadLog(walPath);
|
||||
wal.ReadAll().Any(r => r.Type == WalRecordType.Checkpoint).ShouldBeTrue();
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies restart checkpoint clears WAL and allows subsequent writes.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Checkpoint_Restart_ShouldResetWal_AndAcceptNewWrites()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
db.Users.Insert(new User { Name = "restart-before", Age = 30 });
|
||||
db.SaveChanges();
|
||||
|
||||
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||
var result = db.Checkpoint(CheckpointMode.Restart);
|
||||
result.Mode.ShouldBe(CheckpointMode.Restart);
|
||||
result.Executed.ShouldBeTrue();
|
||||
result.Truncated.ShouldBeTrue();
|
||||
result.Restarted.ShouldBeTrue();
|
||||
db.Storage.GetWalSize().ShouldBe(0);
|
||||
|
||||
db.Users.Insert(new User { Name = "restart-after", Age = 31 });
|
||||
db.SaveChanges();
|
||||
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies recovery remains deterministic after a full checkpoint boundary.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Recover_AfterFullCheckpoint_ShouldApplyLatestCommitDeterministically()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
try
|
||||
{
|
||||
uint pageId;
|
||||
|
||||
using (var storage = new StorageEngine(dbPath, PageFileConfig.Default))
|
||||
{
|
||||
pageId = storage.AllocatePage();
|
||||
|
||||
using (var tx1 = storage.BeginTransaction())
|
||||
{
|
||||
var first = new byte[storage.PageSize];
|
||||
first[0] = 1;
|
||||
storage.WritePage(pageId, tx1.TransactionId, first);
|
||||
tx1.Commit();
|
||||
}
|
||||
|
||||
storage.Checkpoint(CheckpointMode.Full);
|
||||
|
||||
using (var tx2 = storage.BeginTransaction())
|
||||
{
|
||||
var second = new byte[storage.PageSize];
|
||||
second[0] = 2;
|
||||
storage.WritePage(pageId, tx2.TransactionId, second);
|
||||
tx2.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
using (var recovered = new StorageEngine(dbPath, PageFileConfig.Default))
|
||||
{
|
||||
var buffer = new byte[recovered.PageSize];
|
||||
recovered.ReadPage(pageId, 0, buffer);
|
||||
buffer[0].ShouldBe((byte)2);
|
||||
recovered.GetWalSize().ShouldBe(0);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies asynchronous mode-based checkpoints return expected result metadata.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CheckpointAsync_Full_ShouldReturnResult()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
db.Users.Insert(new User { Name = "checkpoint-async", Age = 38 });
|
||||
db.SaveChanges();
|
||||
|
||||
var result = await db.CheckpointAsync(CheckpointMode.Full, TestContext.Current.CancellationToken);
|
||||
result.Mode.ShouldBe(CheckpointMode.Full);
|
||||
result.Executed.ShouldBeTrue();
|
||||
result.Truncated.ShouldBeFalse();
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static SemaphoreSlim GetCommitGate(StorageEngine storage)
|
||||
{
|
||||
var field = typeof(StorageEngine).GetField("_commitLock", BindingFlags.Instance | BindingFlags.NonPublic);
|
||||
field.ShouldNotBeNull();
|
||||
return (SemaphoreSlim)field!.GetValue(storage)!;
|
||||
}
|
||||
|
||||
private static string NewDbPath()
|
||||
=> Path.Combine(Path.GetTempPath(), $"checkpoint_mode_{Guid.NewGuid():N}.db");
|
||||
|
||||
private static void CleanupFiles(string dbPath)
|
||||
{
|
||||
if (File.Exists(dbPath)) File.Delete(dbPath);
|
||||
|
||||
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||
if (File.Exists(walPath)) File.Delete(walPath);
|
||||
|
||||
var markerPath = $"{dbPath}.compact.state";
|
||||
if (File.Exists(markerPath)) File.Delete(markerPath);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user