Implement checkpoint modes with docs/tests and reorganize project file layout
This commit is contained in:
21
README.md
21
README.md
@@ -104,6 +104,27 @@ Operational procedures, diagnostics, and escalation are documented in:
|
|||||||
- [`docs/runbook.md`](docs/runbook.md)
|
- [`docs/runbook.md`](docs/runbook.md)
|
||||||
- [`docs/troubleshooting.md`](docs/troubleshooting.md)
|
- [`docs/troubleshooting.md`](docs/troubleshooting.md)
|
||||||
|
|
||||||
|
## Checkpoint Modes
|
||||||
|
|
||||||
|
`DocumentDbContext` and `StorageEngine` support explicit checkpoint modes via `CheckpointMode`:
|
||||||
|
|
||||||
|
- `Passive`: non-blocking; skips if checkpoint lock is contended.
|
||||||
|
- `Full`: applies committed WAL pages and appends a checkpoint marker, without truncating WAL.
|
||||||
|
- `Truncate`: applies committed WAL pages and truncates WAL.
|
||||||
|
- `Restart`: truncate + WAL writer restart.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```csharp
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
|
var result = db.Checkpoint(CheckpointMode.Full);
|
||||||
|
if (!result.Executed)
|
||||||
|
{
|
||||||
|
// passive checkpoint can be skipped under contention
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
## Security And Compliance Posture
|
## Security And Compliance Posture
|
||||||
|
|
||||||
- CBDD relies on host and process-level access controls.
|
- CBDD relies on host and process-level access controls.
|
||||||
|
|||||||
@@ -29,6 +29,24 @@ Non-goals:
|
|||||||
- `WriteAheadLog`
|
- `WriteAheadLog`
|
||||||
- Storage engine modules under `src/CBDD.Core/Storage`
|
- Storage engine modules under `src/CBDD.Core/Storage`
|
||||||
|
|
||||||
|
Checkpoint APIs:
|
||||||
|
- `DocumentDbContext.Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)`
|
||||||
|
- `DocumentDbContext.CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)`
|
||||||
|
- `StorageEngine.Checkpoint(CheckpointMode mode)`
|
||||||
|
- `StorageEngine.CheckpointAsync(CheckpointMode mode, CancellationToken ct = default)`
|
||||||
|
|
||||||
|
Checkpoint modes:
|
||||||
|
- `Passive`: non-blocking best-effort checkpoint. Returns `Executed = false` when lock is contended.
|
||||||
|
- `Full`: applies committed WAL pages and appends a WAL checkpoint marker without truncating WAL.
|
||||||
|
- `Truncate`: applies committed WAL pages and truncates WAL (default behavior).
|
||||||
|
- `Restart`: same as truncate, then reinitializes WAL writer session.
|
||||||
|
|
||||||
|
Usage guidance:
|
||||||
|
- Use `Passive` for background/low-priority maintenance where latency matters more than immediate WAL cleanup.
|
||||||
|
- Use `Full` when you want durable page-file sync but prefer to preserve WAL history until a later truncate.
|
||||||
|
- Use `Truncate` for routine manual checkpoints and disk-space recovery.
|
||||||
|
- Use `Restart` for aggressive maintenance boundaries (for example after incident remediation flows).
|
||||||
|
|
||||||
## Permissions And Data Handling
|
## Permissions And Data Handling
|
||||||
|
|
||||||
- Database files require host-managed filesystem access controls.
|
- Database files require host-managed filesystem access controls.
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ Database startup fails or recovery path throws WAL/storage errors.
|
|||||||
### Resolution
|
### Resolution
|
||||||
- Pin consumers to last known-good package.
|
- Pin consumers to last known-good package.
|
||||||
- Apply fix and add regression coverage in recovery/transaction tests.
|
- Apply fix and add regression coverage in recovery/transaction tests.
|
||||||
|
- If WAL growth is the issue, run `CheckpointMode.Truncate` (or `Restart`) instead of `Full`.
|
||||||
|
- If foreground latency is a concern, schedule `CheckpointMode.Passive` retries and use `Truncate` during maintenance windows.
|
||||||
|
|
||||||
## Query And Index Issues
|
## Query And Index Issues
|
||||||
|
|
||||||
|
|||||||
@@ -329,6 +329,33 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Executes a checkpoint using the requested mode.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
|
public CheckpointResult Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
|
return Engine.Checkpoint(mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Executes a checkpoint asynchronously using the requested mode.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
|
/// <param name="ct">The cancellation token.</param>
|
||||||
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
|
public Task<CheckpointResult> CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
|
return Engine.CheckpointAsync(mode, ct);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -75,7 +75,7 @@ internal static class BsonExpressionEvaluator
|
|||||||
|
|
||||||
if (name == propertyName)
|
if (name == propertyName)
|
||||||
{
|
{
|
||||||
// Found it! Read value and compare
|
// Found -> read value and compare
|
||||||
return Compare(ref reader, type, targetValue, op);
|
return Compare(ref reader, type, targetValue, op);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -30,148 +30,241 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a checkpoint: merges WAL into PageFile.
|
/// Performs a truncate checkpoint by default.
|
||||||
/// Uses in-memory WAL index for efficiency and consistency.
|
|
||||||
/// </summary>
|
|
||||||
/// <summary>
|
|
||||||
/// Performs a checkpoint: merges WAL into PageFile.
|
|
||||||
/// Uses in-memory WAL index for efficiency and consistency.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Checkpoint()
|
public void Checkpoint()
|
||||||
|
{
|
||||||
|
_ = Checkpoint(CheckpointMode.Truncate);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Performs a checkpoint using the requested mode.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
|
public CheckpointResult Checkpoint(CheckpointMode mode)
|
||||||
|
{
|
||||||
|
bool lockAcquired;
|
||||||
|
if (mode == CheckpointMode.Passive)
|
||||||
|
{
|
||||||
|
lockAcquired = _commitLock.Wait(0);
|
||||||
|
if (!lockAcquired)
|
||||||
|
{
|
||||||
|
var walSize = _wal.GetCurrentSize();
|
||||||
|
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
_commitLock.Wait();
|
_commitLock.Wait();
|
||||||
|
lockAcquired = true;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
CheckpointInternal();
|
return CheckpointInternal(mode);
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
|
{
|
||||||
|
if (lockAcquired)
|
||||||
{
|
{
|
||||||
_commitLock.Release();
|
_commitLock.Release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void CheckpointInternal()
|
private void CheckpointInternal()
|
||||||
{
|
=> _ = CheckpointInternal(CheckpointMode.Truncate);
|
||||||
if (_walIndex.IsEmpty)
|
|
||||||
{
|
|
||||||
// WAL may still contain begin/commit records for read-only transactions.
|
|
||||||
if (_wal.GetCurrentSize() > 0)
|
|
||||||
{
|
|
||||||
_wal.Truncate();
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Write all committed pages from index to PageFile
|
private CheckpointResult CheckpointInternal(CheckpointMode mode)
|
||||||
|
{
|
||||||
|
var walBytesBefore = _wal.GetCurrentSize();
|
||||||
|
var appliedPages = 0;
|
||||||
|
var truncated = false;
|
||||||
|
var restarted = false;
|
||||||
|
|
||||||
|
// 1. Write all committed pages from index to PageFile.
|
||||||
foreach (var kvp in _walIndex)
|
foreach (var kvp in _walIndex)
|
||||||
{
|
{
|
||||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
_pageFile.WritePage(kvp.Key, kvp.Value);
|
||||||
|
appliedPages++;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Flush PageFile to ensure durability
|
// 2. Flush PageFile to ensure durability.
|
||||||
|
if (appliedPages > 0)
|
||||||
|
{
|
||||||
_pageFile.Flush();
|
_pageFile.Flush();
|
||||||
|
}
|
||||||
|
|
||||||
// 3. Clear in-memory WAL index (now persisted)
|
// 3. Clear in-memory WAL index (now persisted).
|
||||||
_walIndex.Clear();
|
_walIndex.Clear();
|
||||||
|
|
||||||
// 4. Truncate WAL (all changes now in PageFile)
|
// 4. Apply mode-specific WAL handling.
|
||||||
|
switch (mode)
|
||||||
|
{
|
||||||
|
case CheckpointMode.Passive:
|
||||||
|
case CheckpointMode.Full:
|
||||||
|
if (walBytesBefore > 0 || appliedPages > 0)
|
||||||
|
{
|
||||||
|
_wal.WriteCheckpointRecord();
|
||||||
|
_wal.Flush();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case CheckpointMode.Truncate:
|
||||||
|
if (walBytesBefore > 0)
|
||||||
|
{
|
||||||
_wal.Truncate();
|
_wal.Truncate();
|
||||||
|
truncated = true;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case CheckpointMode.Restart:
|
||||||
|
_wal.Restart();
|
||||||
|
truncated = true;
|
||||||
|
restarted = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported checkpoint mode.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var walBytesAfter = _wal.GetCurrentSize();
|
||||||
|
return new CheckpointResult(mode, true, appliedPages, walBytesBefore, walBytesAfter, truncated, restarted);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a checkpoint asynchronously by merging WAL pages into the page file.
|
/// Performs a truncate checkpoint asynchronously by default.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
/// <returns>A task that represents the asynchronous checkpoint operation.</returns>
|
|
||||||
public async Task CheckpointAsync(CancellationToken ct = default)
|
public async Task CheckpointAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
_ = await CheckpointAsync(CheckpointMode.Truncate, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Performs a checkpoint asynchronously using the requested mode.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
|
/// <param name="ct">The cancellation token.</param>
|
||||||
|
/// <returns>A task that represents the asynchronous checkpoint operation.</returns>
|
||||||
|
public async Task<CheckpointResult> CheckpointAsync(CheckpointMode mode, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
bool lockAcquired;
|
||||||
|
if (mode == CheckpointMode.Passive)
|
||||||
|
{
|
||||||
|
lockAcquired = await _commitLock.WaitAsync(0, ct);
|
||||||
|
if (!lockAcquired)
|
||||||
|
{
|
||||||
|
var walSize = _wal.GetCurrentSize();
|
||||||
|
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
await _commitLock.WaitAsync(ct);
|
await _commitLock.WaitAsync(ct);
|
||||||
|
lockAcquired = true;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (_walIndex.IsEmpty)
|
// Checkpoint work is synchronous over MMF/page writes for now.
|
||||||
{
|
return CheckpointInternal(mode);
|
||||||
if (_wal.GetCurrentSize() > 0)
|
|
||||||
{
|
|
||||||
_wal.Truncate();
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Write all committed pages from index to PageFile
|
|
||||||
// PageFile writes are sync (MMF), but that's fine as per plan (ValueTask strategy for MMF)
|
|
||||||
foreach (var kvp in _walIndex)
|
|
||||||
{
|
|
||||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Flush PageFile to ensure durability
|
|
||||||
_pageFile.Flush();
|
|
||||||
|
|
||||||
// 3. Clear in-memory WAL index (now persisted)
|
|
||||||
_walIndex.Clear();
|
|
||||||
|
|
||||||
// 4. Truncate WAL (all changes now in PageFile)
|
|
||||||
// WAL truncation involves file resize and flush
|
|
||||||
// TODO: Add TruncateAsync to WAL? For now Truncate is sync.
|
|
||||||
_wal.Truncate();
|
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
|
{
|
||||||
|
if (lockAcquired)
|
||||||
{
|
{
|
||||||
_commitLock.Release();
|
_commitLock.Release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Recovers from crash by replaying WAL.
|
/// Recovers from crash by replaying WAL.
|
||||||
/// Applies all committed transactions to PageFile, then truncates WAL.
|
/// Applies committed transactions to PageFile in deterministic WAL order, then truncates WAL.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Recover()
|
public void Recover()
|
||||||
{
|
{
|
||||||
_commitLock.Wait();
|
_commitLock.Wait();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// 1. Read WAL and identify committed transactions
|
// 1. Read WAL and locate the latest checkpoint boundary.
|
||||||
var records = _wal.ReadAll();
|
var records = _wal.ReadAll();
|
||||||
var committedTxns = new HashSet<ulong>();
|
var startIndex = 0;
|
||||||
var txnWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
for (var i = records.Count - 1; i >= 0; i--)
|
||||||
|
|
||||||
foreach (var record in records)
|
|
||||||
{
|
{
|
||||||
if (record.Type == WalRecordType.Commit)
|
if (records[i].Type == WalRecordType.Checkpoint)
|
||||||
committedTxns.Add(record.TransactionId);
|
|
||||||
else if (record.Type == WalRecordType.Write)
|
|
||||||
{
|
{
|
||||||
if (!txnWrites.ContainsKey(record.TransactionId))
|
startIndex = i + 1;
|
||||||
txnWrites[record.TransactionId] = new List<(uint, byte[])>();
|
break;
|
||||||
|
|
||||||
if (record.AfterImage != null)
|
|
||||||
{
|
|
||||||
txnWrites[record.TransactionId].Add((record.PageId, record.AfterImage));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 2. Replay WAL in source order with deterministic commit application.
|
||||||
|
var pendingWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
||||||
|
var appliedAny = false;
|
||||||
|
|
||||||
|
for (var i = startIndex; i < records.Count; i++)
|
||||||
|
{
|
||||||
|
var record = records[i];
|
||||||
|
switch (record.Type)
|
||||||
|
{
|
||||||
|
case WalRecordType.Begin:
|
||||||
|
if (!pendingWrites.ContainsKey(record.TransactionId))
|
||||||
|
{
|
||||||
|
pendingWrites[record.TransactionId] = new List<(uint, byte[])>();
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case WalRecordType.Write:
|
||||||
|
if (record.AfterImage == null)
|
||||||
|
{
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Apply committed transactions to PageFile
|
if (!pendingWrites.TryGetValue(record.TransactionId, out var writes))
|
||||||
foreach (var txnId in committedTxns)
|
|
||||||
{
|
{
|
||||||
if (!txnWrites.ContainsKey(txnId))
|
writes = new List<(uint, byte[])>();
|
||||||
continue;
|
pendingWrites[record.TransactionId] = writes;
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var (pageId, data) in txnWrites[txnId])
|
writes.Add((record.PageId, record.AfterImage));
|
||||||
|
break;
|
||||||
|
case WalRecordType.Commit:
|
||||||
|
if (!pendingWrites.TryGetValue(record.TransactionId, out var committedWrites))
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var (pageId, data) in committedWrites)
|
||||||
{
|
{
|
||||||
_pageFile.WritePage(pageId, data);
|
_pageFile.WritePage(pageId, data);
|
||||||
|
appliedAny = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
pendingWrites.Remove(record.TransactionId);
|
||||||
|
break;
|
||||||
|
case WalRecordType.Abort:
|
||||||
|
pendingWrites.Remove(record.TransactionId);
|
||||||
|
break;
|
||||||
|
case WalRecordType.Checkpoint:
|
||||||
|
pendingWrites.Clear();
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Flush PageFile to ensure durability
|
// 3. Flush PageFile to ensure durability.
|
||||||
|
if (appliedAny)
|
||||||
|
{
|
||||||
_pageFile.Flush();
|
_pageFile.Flush();
|
||||||
|
}
|
||||||
|
|
||||||
// 4. Clear in-memory WAL index (redundant since we just recovered)
|
// 4. Clear in-memory WAL index (redundant since we just recovered).
|
||||||
_walIndex.Clear();
|
_walIndex.Clear();
|
||||||
|
|
||||||
// 5. Truncate WAL (all changes now in PageFile)
|
// 5. Truncate WAL (all changes now in PageFile).
|
||||||
|
if (_wal.GetCurrentSize() > 0)
|
||||||
|
{
|
||||||
_wal.Truncate();
|
_wal.Truncate();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
_commitLock.Release();
|
_commitLock.Release();
|
||||||
|
|||||||
@@ -7,26 +7,46 @@ namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
|||||||
public enum CheckpointMode
|
public enum CheckpointMode
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Passive checkpoint: Non-blocking, best-effort transfer from WAL to database.
|
/// Passive checkpoint: non-blocking, best-effort transfer from WAL to database.
|
||||||
/// Does not wait for readers or writers. May not checkpoint all frames.
|
/// If the checkpoint lock is busy, the operation is skipped.
|
||||||
|
/// WAL content is preserved and a checkpoint marker is appended when work is applied.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Passive = 0,
|
Passive = 0,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Full checkpoint: Waits for concurrent readers/writers, then checkpoints all
|
/// Full checkpoint: waits for the checkpoint lock, transfers committed pages to
|
||||||
/// committed transactions from WAL to database. Blocks until complete.
|
/// the page file, and preserves WAL content by appending a checkpoint marker.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Full = 1,
|
Full = 1,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Truncate checkpoint: Same as Full, but also truncates the WAL file after
|
/// Truncate checkpoint: same as <see cref="Full"/> but truncates WAL after
|
||||||
/// successful checkpoint. Use this to reclaim disk space.
|
/// successfully applying committed pages. Use this to reclaim disk space.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Truncate = 2,
|
Truncate = 2,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Restart checkpoint: Truncates WAL and restarts with a new WAL file.
|
/// Restart checkpoint: same as <see cref="Truncate"/> and then reinitializes
|
||||||
/// Forces a fresh start. Most aggressive mode.
|
/// the WAL stream for a fresh writer session.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Restart = 3
|
Restart = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Result of a checkpoint execution.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="Mode">Requested checkpoint mode.</param>
|
||||||
|
/// <param name="Executed">True when checkpoint logic ran; false when skipped (for passive mode contention).</param>
|
||||||
|
/// <param name="AppliedPages">Number of pages copied from WAL index to page file.</param>
|
||||||
|
/// <param name="WalBytesBefore">WAL size before the operation.</param>
|
||||||
|
/// <param name="WalBytesAfter">WAL size after the operation.</param>
|
||||||
|
/// <param name="Truncated">True when WAL was truncated by this operation.</param>
|
||||||
|
/// <param name="Restarted">True when WAL stream was restarted by this operation.</param>
|
||||||
|
public readonly record struct CheckpointResult(
|
||||||
|
CheckpointMode Mode,
|
||||||
|
bool Executed,
|
||||||
|
int AppliedPages,
|
||||||
|
long WalBytesBefore,
|
||||||
|
long WalBytesAfter,
|
||||||
|
bool Truncated,
|
||||||
|
bool Restarted);
|
||||||
|
|||||||
@@ -225,6 +225,61 @@ public sealed class WriteAheadLog : IDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes a checkpoint marker record.
|
||||||
|
/// </summary>
|
||||||
|
public void WriteCheckpointRecord()
|
||||||
|
{
|
||||||
|
_lock.Wait();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
WriteCheckpointRecordInternal();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes a checkpoint marker record asynchronously.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="ct">The cancellation token.</param>
|
||||||
|
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||||
|
public async ValueTask WriteCheckpointRecordAsync(CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
await _lock.WaitAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(17);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
buffer[0] = (byte)WalRecordType.Checkpoint;
|
||||||
|
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), 0UL);
|
||||||
|
BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||||
|
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, 17), ct);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WriteCheckpointRecordInternal()
|
||||||
|
{
|
||||||
|
Span<byte> buffer = stackalloc byte[17]; // type(1) + reserved(8) + timestamp(8)
|
||||||
|
buffer[0] = (byte)WalRecordType.Checkpoint;
|
||||||
|
BitConverter.TryWriteBytes(buffer[1..9], 0UL);
|
||||||
|
BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||||
|
_walStream!.Write(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a data modification record
|
/// Writes a data modification record
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@@ -399,6 +454,28 @@ public sealed class WriteAheadLog : IDisposable
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Truncates and reopens the WAL stream to start a fresh writer session.
|
||||||
|
/// </summary>
|
||||||
|
public void Restart()
|
||||||
|
{
|
||||||
|
_lock.Wait();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_walStream?.Dispose();
|
||||||
|
_walStream = new FileStream(
|
||||||
|
_walPath,
|
||||||
|
FileMode.Create,
|
||||||
|
FileAccess.ReadWrite,
|
||||||
|
FileShare.None,
|
||||||
|
bufferSize: 64 * 1024);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_lock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads all WAL records (for recovery)
|
/// Reads all WAL records (for recovery)
|
||||||
@@ -440,6 +517,7 @@ public sealed class WriteAheadLog : IDisposable
|
|||||||
case WalRecordType.Begin:
|
case WalRecordType.Begin:
|
||||||
case WalRecordType.Commit:
|
case WalRecordType.Commit:
|
||||||
case WalRecordType.Abort:
|
case WalRecordType.Abort:
|
||||||
|
case WalRecordType.Checkpoint:
|
||||||
// Read common fields (txnId + timestamp = 16 bytes)
|
// Read common fields (txnId + timestamp = 16 bytes)
|
||||||
var bytesRead = _walStream.Read(headerBuf);
|
var bytesRead = _walStream.Read(headerBuf);
|
||||||
if (bytesRead < 16)
|
if (bytesRead < 16)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>netstandard2.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
||||||
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
||||||
<LangVersion>latest</LangVersion>
|
<LangVersion>latest</LangVersion>
|
||||||
@@ -29,8 +29,6 @@
|
|||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
|
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
|
||||||
|
|
||||||
<None Include="_._" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ using ZB.MOM.WW.CBDD.Core.Compression;
|
|||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
using ZB.MOM.WW.CBDD.Core.Metadata;
|
using ZB.MOM.WW.CBDD.Core.Metadata;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Shared;
|
namespace ZB.MOM.WW.CBDD.Shared;
|
||||||
|
|
||||||
@@ -243,6 +244,15 @@ public partial class TestDbContext : DocumentDbContext
|
|||||||
Engine.Checkpoint();
|
Engine.Checkpoint();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Executes ForceCheckpoint with the requested checkpoint mode.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
|
public CheckpointResult ForceCheckpoint(CheckpointMode mode)
|
||||||
|
{
|
||||||
|
return Engine.Checkpoint(mode);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Storage.
|
/// Gets or sets the Storage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
228
tests/CBDD.Tests/Storage/CheckpointModeTests.cs
Normal file
228
tests/CBDD.Tests/Storage/CheckpointModeTests.cs
Normal file
@@ -0,0 +1,228 @@
|
|||||||
|
using System.Reflection;
|
||||||
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
using ZB.MOM.WW.CBDD.Shared;
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Tests;
|
||||||
|
|
||||||
|
public class CheckpointModeTests
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies default checkpoint mode truncates WAL.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Checkpoint_Default_ShouldUseTruncate()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var db = new TestDbContext(dbPath);
|
||||||
|
db.Users.Insert(new User { Name = "checkpoint-default", Age = 42 });
|
||||||
|
db.SaveChanges();
|
||||||
|
|
||||||
|
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||||
|
var result = db.Checkpoint();
|
||||||
|
|
||||||
|
result.Mode.ShouldBe(CheckpointMode.Truncate);
|
||||||
|
result.Executed.ShouldBeTrue();
|
||||||
|
result.Truncated.ShouldBeTrue();
|
||||||
|
db.Storage.GetWalSize().ShouldBe(0);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies passive mode skips when checkpoint lock is contended.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Checkpoint_Passive_ShouldSkip_WhenLockIsContended()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var storage = new StorageEngine(dbPath, PageFileConfig.Default);
|
||||||
|
var gate = GetCommitGate(storage);
|
||||||
|
gate.Wait(TestContext.Current.CancellationToken);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = storage.Checkpoint(CheckpointMode.Passive);
|
||||||
|
result.Mode.ShouldBe(CheckpointMode.Passive);
|
||||||
|
result.Executed.ShouldBeFalse();
|
||||||
|
result.Truncated.ShouldBeFalse();
|
||||||
|
result.Restarted.ShouldBeFalse();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
gate.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies full checkpoint applies data and appends a checkpoint marker without truncating WAL.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Checkpoint_Full_ShouldAppendMarker_AndPreserveWal()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using (var db = new TestDbContext(dbPath))
|
||||||
|
{
|
||||||
|
db.Users.Insert(new User { Name = "checkpoint-full", Age = 50 });
|
||||||
|
db.SaveChanges();
|
||||||
|
|
||||||
|
var walBefore = db.Storage.GetWalSize();
|
||||||
|
walBefore.ShouldBeGreaterThan(0);
|
||||||
|
|
||||||
|
var result = db.Checkpoint(CheckpointMode.Full);
|
||||||
|
result.Mode.ShouldBe(CheckpointMode.Full);
|
||||||
|
result.Executed.ShouldBeTrue();
|
||||||
|
result.Truncated.ShouldBeFalse();
|
||||||
|
result.WalBytesAfter.ShouldBeGreaterThan(0);
|
||||||
|
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
using var wal = new WriteAheadLog(walPath);
|
||||||
|
wal.ReadAll().Any(r => r.Type == WalRecordType.Checkpoint).ShouldBeTrue();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies restart checkpoint clears WAL and allows subsequent writes.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Checkpoint_Restart_ShouldResetWal_AndAcceptNewWrites()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var db = new TestDbContext(dbPath);
|
||||||
|
db.Users.Insert(new User { Name = "restart-before", Age = 30 });
|
||||||
|
db.SaveChanges();
|
||||||
|
|
||||||
|
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||||
|
var result = db.Checkpoint(CheckpointMode.Restart);
|
||||||
|
result.Mode.ShouldBe(CheckpointMode.Restart);
|
||||||
|
result.Executed.ShouldBeTrue();
|
||||||
|
result.Truncated.ShouldBeTrue();
|
||||||
|
result.Restarted.ShouldBeTrue();
|
||||||
|
db.Storage.GetWalSize().ShouldBe(0);
|
||||||
|
|
||||||
|
db.Users.Insert(new User { Name = "restart-after", Age = 31 });
|
||||||
|
db.SaveChanges();
|
||||||
|
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies recovery remains deterministic after a full checkpoint boundary.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public void Recover_AfterFullCheckpoint_ShouldApplyLatestCommitDeterministically()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
uint pageId;
|
||||||
|
|
||||||
|
using (var storage = new StorageEngine(dbPath, PageFileConfig.Default))
|
||||||
|
{
|
||||||
|
pageId = storage.AllocatePage();
|
||||||
|
|
||||||
|
using (var tx1 = storage.BeginTransaction())
|
||||||
|
{
|
||||||
|
var first = new byte[storage.PageSize];
|
||||||
|
first[0] = 1;
|
||||||
|
storage.WritePage(pageId, tx1.TransactionId, first);
|
||||||
|
tx1.Commit();
|
||||||
|
}
|
||||||
|
|
||||||
|
storage.Checkpoint(CheckpointMode.Full);
|
||||||
|
|
||||||
|
using (var tx2 = storage.BeginTransaction())
|
||||||
|
{
|
||||||
|
var second = new byte[storage.PageSize];
|
||||||
|
second[0] = 2;
|
||||||
|
storage.WritePage(pageId, tx2.TransactionId, second);
|
||||||
|
tx2.Commit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
using (var recovered = new StorageEngine(dbPath, PageFileConfig.Default))
|
||||||
|
{
|
||||||
|
var buffer = new byte[recovered.PageSize];
|
||||||
|
recovered.ReadPage(pageId, 0, buffer);
|
||||||
|
buffer[0].ShouldBe((byte)2);
|
||||||
|
recovered.GetWalSize().ShouldBe(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Verifies asynchronous mode-based checkpoints return expected result metadata.
|
||||||
|
/// </summary>
|
||||||
|
[Fact]
|
||||||
|
public async Task CheckpointAsync_Full_ShouldReturnResult()
|
||||||
|
{
|
||||||
|
var dbPath = NewDbPath();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var db = new TestDbContext(dbPath);
|
||||||
|
db.Users.Insert(new User { Name = "checkpoint-async", Age = 38 });
|
||||||
|
db.SaveChanges();
|
||||||
|
|
||||||
|
var result = await db.CheckpointAsync(CheckpointMode.Full, TestContext.Current.CancellationToken);
|
||||||
|
result.Mode.ShouldBe(CheckpointMode.Full);
|
||||||
|
result.Executed.ShouldBeTrue();
|
||||||
|
result.Truncated.ShouldBeFalse();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
CleanupFiles(dbPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SemaphoreSlim GetCommitGate(StorageEngine storage)
|
||||||
|
{
|
||||||
|
var field = typeof(StorageEngine).GetField("_commitLock", BindingFlags.Instance | BindingFlags.NonPublic);
|
||||||
|
field.ShouldNotBeNull();
|
||||||
|
return (SemaphoreSlim)field!.GetValue(storage)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string NewDbPath()
|
||||||
|
=> Path.Combine(Path.GetTempPath(), $"checkpoint_mode_{Guid.NewGuid():N}.db");
|
||||||
|
|
||||||
|
private static void CleanupFiles(string dbPath)
|
||||||
|
{
|
||||||
|
if (File.Exists(dbPath)) File.Delete(dbPath);
|
||||||
|
|
||||||
|
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||||
|
if (File.Exists(walPath)) File.Delete(walPath);
|
||||||
|
|
||||||
|
var markerPath = $"{dbPath}.compact.state";
|
||||||
|
if (File.Exists(markerPath)) File.Delete(markerPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user