dohertj2/CBDD
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reformat / cleanup
All checks were successful
NuGet Publish / build-and-pack (push) Successful in 46s
Details
NuGet Publish / publish-to-gitea (push) Successful in 56s
Details

Browse Source
This commit is contained in:
Joseph Doherty
2026-02-21 08:10:36 -05:00
parent 4c6aaa5a3f
commit a70d8befae
176 changed files with 50555 additions and 49587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
tests/CBDD.Tests/Compaction/CompactionCrashRecoveryTests.cs
View File

@@ -7,7 +7,7 @@ namespace ZB.MOM.WW.CBDD.Tests;
public class CompactionCrashRecoveryTests
{
/// <summary>
/// Verifies compaction resumes from marker phases and preserves data.
/// Verifies compaction resumes from marker phases and preserves data.
/// </summary>
/// <param name="phase">The crash marker phase to resume from.</param>
[Theory]
@@ -16,8 +16,8 @@ public class CompactionCrashRecoveryTests
[InlineData("Swapped")]
public void ResumeCompaction_FromCrashMarkerPhases_ShouldFinalizeAndPreserveData(string phase)
{
var dbPath = NewDbPath();
var markerPath = MarkerPath(dbPath);
string dbPath = NewDbPath();
string markerPath = MarkerPath(dbPath);
try
{
@@ -54,13 +54,13 @@ public class CompactionCrashRecoveryTests
}
/// <summary>
/// Verifies corrupted compaction markers are recovered deterministically.
/// Verifies corrupted compaction markers are recovered deterministically.
/// </summary>
[Fact]
public void ResumeCompaction_WithCorruptedMarker_ShouldRecoverDeterministically()
{
var dbPath = NewDbPath();
var markerPath = MarkerPath(dbPath);
string dbPath = NewDbPath();
string markerPath = MarkerPath(dbPath);
try
{
@@ -96,13 +96,11 @@ public class CompactionCrashRecoveryTests
{
var ids = new List<ObjectId>();
for (var i = 0; i < 120; i++)
{
ids.Add(db.Users.Insert(new User
{
Name = $"user-{i:D4}-payload-{new string('x', 120)}",
Age = i % 20
}));
}
db.SaveChanges();
return ids;
@@ -110,25 +108,30 @@ public class CompactionCrashRecoveryTests
private static void WriteMarker(string markerPath, string dbPath, string phase)
{
var safeDbPath = dbPath.Replace("\\", "\\\\", StringComparison.Ordinal);
string safeDbPath = dbPath.Replace("\\", "\\\\", StringComparison.Ordinal);
var now = DateTimeOffset.UtcNow.ToString("O");
var json = $$"""
{"version":1,"phase":"{{phase}}","databasePath":"{{safeDbPath}}","startedAtUtc":"{{now}}","lastUpdatedUtc":"{{now}}","onlineMode":false,"mode":"InPlace"}
""";
{"version":1,"phase":"{{phase}}","databasePath":"{{safeDbPath}}","startedAtUtc":"{{now}}","lastUpdatedUtc":"{{now}}","onlineMode":false,"mode":"InPlace"}
""";
File.WriteAllText(markerPath, json);
}
private static string MarkerPath(string dbPath) => $"{dbPath}.compact.state";
private static string MarkerPath(string dbPath)
{
return $"{dbPath}.compact.state";
}
private static string NewDbPath()
=> Path.Combine(Path.GetTempPath(), $"compaction_crash_{Guid.NewGuid():N}.db");
{
return Path.Combine(Path.GetTempPath(), $"compaction_crash_{Guid.NewGuid():N}.db");
}
private static void CleanupFiles(string dbPath)
{
var walPath = Path.ChangeExtension(dbPath, ".wal");
var markerPath = MarkerPath(dbPath);
string walPath = Path.ChangeExtension(dbPath, ".wal");
string markerPath = MarkerPath(dbPath);
if (File.Exists(dbPath)) File.Delete(dbPath);
if (File.Exists(walPath)) File.Delete(walPath);
if (File.Exists(markerPath)) File.Delete(markerPath);
}
}
}

101
tests/CBDD.Tests/Compaction/CompactionOfflineTests.cs
View File

@@ -1,4 +1,5 @@
using System.IO.MemoryMappedFiles;
using System.Text;
using ZB.MOM.WW.CBDD.Bson;
using ZB.MOM.WW.CBDD.Core.Indexing;
using ZB.MOM.WW.CBDD.Core.Storage;
@@ -9,30 +10,23 @@ namespace ZB.MOM.WW.CBDD.Tests;
public class CompactionOfflineTests
{
/// <summary>
/// Tests offline compact should preserve logical data equivalence.
/// Tests offline compact should preserve logical data equivalence.
/// </summary>
[Fact]
public void OfflineCompact_ShouldPreserveLogicalDataEquivalence()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
using var db = new TestDbContext(dbPath);
var ids = new List<ObjectId>();
for (var i = 0; i < 160; i++)
{
ids.Add(db.Users.Insert(new User { Name = $"user-{i:D4}", Age = i % 31 }));
}
for (var i = 0; i < 160; i++) ids.Add(db.Users.Insert(new User { Name = $"user-{i:D4}", Age = i % 31 }));
for (var i = 0; i < ids.Count; i += 9)
{
if (db.Users.FindById(ids[i]) != null)
{
db.Users.Delete(ids[i]).ShouldBeTrue();
}
}
var updateTargets = db.Users.FindAll(u => u.Age % 4 == 0)
.Select(u => u.Id)
@@ -40,10 +34,7 @@ public class CompactionOfflineTests
foreach (var id in updateTargets)
{
var user = db.Users.FindById(id);
if (user == null)
{
continue;
}
if (user == null) continue;
user.Name += "-updated";
db.Users.Update(user).ShouldBeTrue();
@@ -76,25 +67,23 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact should keep index results consistent.
/// Tests offline compact should keep index results consistent.
/// </summary>
[Fact]
public void OfflineCompact_ShouldKeepIndexResultsConsistent()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
using var db = new TestDbContext(dbPath);
for (var i = 0; i < 300; i++)
{
db.People.Insert(new Person
{
Name = $"person-{i:D4}",
Age = i % 12
});
}
db.SaveChanges();
db.ForceCheckpoint();
@@ -104,7 +93,7 @@ public class CompactionOfflineTests
.ToDictionary(g => g.Key, g => g.Select(x => x.Name).OrderBy(x => x).ToArray());
db.SaveChanges();
var indexNamesBefore = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
string[] indexNamesBefore = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
var stats = db.Compact(new CompactionOptions
{
@@ -114,12 +103,12 @@ public class CompactionOfflineTests
});
stats.PrePageCount.ShouldBeGreaterThanOrEqualTo(stats.PostPageCount);
var indexNamesAfter = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
string[] indexNamesAfter = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
indexNamesAfter.ShouldBe(indexNamesBefore);
foreach (var age in expectedByAge.Keys.OrderBy(x => x))
foreach (int age in expectedByAge.Keys.OrderBy(x => x))
{
var actual = db.People.FindAll(p => p.Age == age)
string[] actual = db.People.FindAll(p => p.Age == age)
.Select(x => x.Name)
.OrderBy(x => x)
.ToArray();
@@ -134,25 +123,23 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact should rebuild hash index metadata and preserve results.
/// Tests offline compact should rebuild hash index metadata and preserve results.
/// </summary>
[Fact]
public void OfflineCompact_ShouldRebuildHashIndexMetadataAndPreserveResults()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
using var db = new TestDbContext(dbPath);
for (var i = 0; i < 300; i++)
{
db.People.Insert(new Person
{
Name = $"hash-person-{i:D4}",
Age = i % 12
});
}
db.SaveChanges();
db.ForceCheckpoint();
@@ -165,7 +152,8 @@ public class CompactionOfflineTests
metadata.ShouldNotBeNull();
var targetIndex = metadata!.Indexes
.FirstOrDefault(index => index.PropertyPaths.Any(path => path.Equals("Age", StringComparison.OrdinalIgnoreCase)));
.FirstOrDefault(index =>
index.PropertyPaths.Any(path => path.Equals("Age", StringComparison.OrdinalIgnoreCase)));
targetIndex.ShouldNotBeNull();
targetIndex!.Type = IndexType.Hash;
@@ -191,9 +179,9 @@ public class CompactionOfflineTests
runtimeIndex.ShouldNotBeNull();
runtimeIndex!.Type.ShouldBe(IndexType.Hash);
foreach (var age in expectedByAge.Keys.OrderBy(x => x))
foreach (int age in expectedByAge.Keys.OrderBy(x => x))
{
var actual = db.People.FindAll(p => p.Age == age)
string[] actual = db.People.FindAll(p => p.Age == age)
.Select(x => x.Name)
.OrderBy(x => x)
.ToArray();
@@ -208,12 +196,12 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact when tail is reclaimable should reduce file size.
/// Tests offline compact when tail is reclaimable should reduce file size.
/// </summary>
[Fact]
public void OfflineCompact_WhenTailIsReclaimable_ShouldReduceFileSize()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
var ids = new List<ObjectId>();
try
@@ -233,24 +221,20 @@ public class CompactionOfflineTests
db.SaveChanges();
db.ForceCheckpoint();
for (var i = ids.Count - 1; i >= 60; i--)
{
for (int i = ids.Count - 1; i >= 60; i--)
if (db.Users.FindById(ids[i]) != null)
{
db.Users.Delete(ids[i]).ShouldBeTrue();
}
}
db.SaveChanges();
db.ForceCheckpoint();
var preCompactSize = new FileInfo(dbPath).Length;
long preCompactSize = new FileInfo(dbPath).Length;
var stats = db.Compact(new CompactionOptions
{
EnableTailTruncation = true,
MinimumRetainedPages = 2
});
var postCompactSize = new FileInfo(dbPath).Length;
long postCompactSize = new FileInfo(dbPath).Length;
postCompactSize.ShouldBeLessThanOrEqualTo(preCompactSize);
stats.ReclaimedFileBytes.ShouldBeGreaterThanOrEqualTo(0);
@@ -262,20 +246,17 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact with invalid primary root metadata should fail validation.
/// Tests offline compact with invalid primary root metadata should fail validation.
/// </summary>
[Fact]
public void OfflineCompact_WithInvalidPrimaryRootMetadata_ShouldFailValidation()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
using var db = new TestDbContext(dbPath);
for (var i = 0; i < 32; i++)
{
db.Users.Insert(new User { Name = $"invalid-primary-{i:D3}", Age = i });
}
for (var i = 0; i < 32; i++) db.Users.Insert(new User { Name = $"invalid-primary-{i:D3}", Age = i });
db.SaveChanges();
db.ForceCheckpoint();
@@ -295,20 +276,18 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact with invalid secondary root metadata should fail validation.
/// Tests offline compact with invalid secondary root metadata should fail validation.
/// </summary>
[Fact]
public void OfflineCompact_WithInvalidSecondaryRootMetadata_ShouldFailValidation()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
using var db = new TestDbContext(dbPath);
for (var i = 0; i < 48; i++)
{
db.People.Insert(new Person { Name = $"invalid-secondary-{i:D3}", Age = i % 10 });
}
db.SaveChanges();
db.ForceCheckpoint();
@@ -329,12 +308,12 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact should report live bytes relocation and throughput telemetry.
/// Tests offline compact should report live bytes relocation and throughput telemetry.
/// </summary>
[Fact]
public void OfflineCompact_ShouldReportLiveBytesRelocationAndThroughputTelemetry()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
@@ -342,21 +321,15 @@ public class CompactionOfflineTests
var ids = new List<ObjectId>();
for (var i = 0; i < 160; i++)
{
ids.Add(db.Users.Insert(new User
{
Name = BuildPayload(i, 9_000),
Age = i
}));
}
for (var i = 0; i < ids.Count; i += 7)
{
if (db.Users.FindById(ids[i]) != null)
{
db.Users.Delete(ids[i]).ShouldBeTrue();
}
}
db.SaveChanges();
db.ForceCheckpoint();
@@ -383,12 +356,12 @@ public class CompactionOfflineTests
}
/// <summary>
/// Tests offline compact when primary index points to deleted slot should fail validation.
/// Tests offline compact when primary index points to deleted slot should fail validation.
/// </summary>
[Fact]
public void OfflineCompact_WhenPrimaryIndexPointsToDeletedSlot_ShouldFailValidation()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
try
{
@@ -408,7 +381,7 @@ public class CompactionOfflineTests
db.Storage.ReadPage(location.PageId, null, page);
var header = SlottedPageHeader.ReadFrom(page);
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
int slotOffset = SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size;
var slot = SlotEntry.ReadFrom(page.AsSpan(slotOffset, SlotEntry.Size));
slot.Flags |= SlotFlags.Deleted;
slot.WriteTo(page.AsSpan(slotOffset, SlotEntry.Size));
@@ -441,7 +414,7 @@ public class CompactionOfflineTests
private static string BuildPayload(int seed, int approxLength)
{
var builder = new System.Text.StringBuilder(approxLength + 256);
var builder = new StringBuilder(approxLength + 256);
var i = 0;
while (builder.Length < approxLength)
{
@@ -457,11 +430,13 @@ public class CompactionOfflineTests
}
private static string NewDbPath()
=> Path.Combine(Path.GetTempPath(), $"compaction_offline_{Guid.NewGuid():N}.db");
{
return Path.Combine(Path.GetTempPath(), $"compaction_offline_{Guid.NewGuid():N}.db");
}
private static void CleanupFiles(string dbPath)
{
var walPath = Path.ChangeExtension(dbPath, ".wal");
string walPath = Path.ChangeExtension(dbPath, ".wal");
var markerPath = $"{dbPath}.compact.state";
var tempPath = $"{dbPath}.compact.tmp";
var backupPath = $"{dbPath}.compact.bak";
@@ -471,4 +446,4 @@ public class CompactionOfflineTests
if (File.Exists(tempPath)) File.Delete(tempPath);
if (File.Exists(backupPath)) File.Delete(backupPath);
}
}
}

27
tests/CBDD.Tests/Compaction/CompactionOnlineConcurrencyTests.cs
View File

@@ -7,12 +7,12 @@ namespace ZB.MOM.WW.CBDD.Tests;
public class CompactionOnlineConcurrencyTests
{
/// <summary>
/// Verifies online compaction completes without deadlock under concurrent workload.
/// Verifies online compaction completes without deadlock under concurrent workload.
/// </summary>
[Fact]
public async Task OnlineCompaction_WithConcurrentishWorkload_ShouldCompleteWithoutDeadlock()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
var activeIds = new List<ObjectId>();
var sync = new object();
var completedOps = 0;
@@ -48,10 +48,7 @@ public class CompactionOnlineConcurrencyTests
ObjectId? candidate = null;
lock (sync)
{
if (activeIds.Count > 0)
{
candidate = activeIds[i % activeIds.Count];
}
if (activeIds.Count > 0) candidate = activeIds[i % activeIds.Count];
}
if (candidate.HasValue)
@@ -76,10 +73,7 @@ public class CompactionOnlineConcurrencyTests
}
}
if (candidate.HasValue)
{
db.Users.Delete(candidate.Value);
}
if (candidate.HasValue) db.Users.Delete(candidate.Value);
}
db.SaveChanges();
@@ -115,10 +109,7 @@ public class CompactionOnlineConcurrencyTests
}
var actualIds = allUsers.Select(x => x.Id).ToHashSet();
foreach (var id in snapshotIds)
{
actualIds.ShouldContain(id);
}
foreach (var id in snapshotIds) actualIds.ShouldContain(id);
}
finally
{
@@ -127,14 +118,16 @@ public class CompactionOnlineConcurrencyTests
}
private static string NewDbPath()
=> Path.Combine(Path.GetTempPath(), $"compaction_online_{Guid.NewGuid():N}.db");
{
return Path.Combine(Path.GetTempPath(), $"compaction_online_{Guid.NewGuid():N}.db");
}
private static void CleanupFiles(string dbPath)
{
var walPath = Path.ChangeExtension(dbPath, ".wal");
string walPath = Path.ChangeExtension(dbPath, ".wal");
var markerPath = $"{dbPath}.compact.state";
if (File.Exists(dbPath)) File.Delete(dbPath);
if (File.Exists(walPath)) File.Delete(walPath);
if (File.Exists(markerPath)) File.Delete(markerPath);
}
}
}

33
tests/CBDD.Tests/Compaction/CompactionWalCoordinationTests.cs
View File

@@ -7,21 +7,18 @@ namespace ZB.MOM.WW.CBDD.Tests;
public class CompactionWalCoordinationTests
{
/// <summary>
/// Verifies offline compaction checkpoints and leaves the WAL empty.
/// Verifies offline compaction checkpoints and leaves the WAL empty.
/// </summary>
[Fact]
public void OfflineCompact_ShouldCheckpointAndLeaveWalEmpty()
{
var dbPath = NewDbPath();
string dbPath = NewDbPath();
var markerPath = $"{dbPath}.compact.state";
try
{
using var db = new TestDbContext(dbPath);
for (var i = 0; i < 80; i++)
{
db.Users.Insert(new User { Name = $"wal-compact-{i:D3}", Age = i });
}
for (var i = 0; i < 80; i++) db.Users.Insert(new User { Name = $"wal-compact-{i:D3}", Age = i });
db.SaveChanges();
db.Storage.GetWalSize().ShouldBeGreaterThan(0);
@@ -46,13 +43,13 @@ public class CompactionWalCoordinationTests
}
/// <summary>
/// Verifies compaction after WAL recovery preserves durable data.
/// Verifies compaction after WAL recovery preserves durable data.
/// </summary>
[Fact]
public void Compact_AfterWalRecovery_ShouldKeepDataDurable()
{
var dbPath = NewDbPath();
var walPath = Path.ChangeExtension(dbPath, ".wal");
string dbPath = NewDbPath();
string walPath = Path.ChangeExtension(dbPath, ".wal");
var expected = new List<(ObjectId Id, string Name)>();
try
@@ -76,10 +73,7 @@ public class CompactionWalCoordinationTests
{
recovered.Users.Count().ShouldBe(expected.Count);
foreach (var item in expected)
{
recovered.Users.FindById(item.Id)!.Name.ShouldBe(item.Name);
}
foreach (var item in expected) recovered.Users.FindById(item.Id)!.Name.ShouldBe(item.Name);
recovered.SaveChanges();
recovered.Compact();
@@ -89,10 +83,7 @@ public class CompactionWalCoordinationTests
using (var verify = new TestDbContext(dbPath))
{
verify.Users.Count().ShouldBe(expected.Count);
foreach (var item in expected)
{
verify.Users.FindById(item.Id)!.Name.ShouldBe(item.Name);
}
foreach (var item in expected) verify.Users.FindById(item.Id)!.Name.ShouldBe(item.Name);
}
}
finally
@@ -102,14 +93,16 @@ public class CompactionWalCoordinationTests
}
private static string NewDbPath()
=> Path.Combine(Path.GetTempPath(), $"compaction_wal_{Guid.NewGuid():N}.db");
{
return Path.Combine(Path.GetTempPath(), $"compaction_wal_{Guid.NewGuid():N}.db");
}
private static void CleanupFiles(string dbPath)
{
var walPath = Path.ChangeExtension(dbPath, ".wal");
string walPath = Path.ChangeExtension(dbPath, ".wal");
var markerPath = $"{dbPath}.compact.state";
if (File.Exists(dbPath)) File.Delete(dbPath);
if (File.Exists(walPath)) File.Delete(walPath);
if (File.Exists(markerPath)) File.Delete(markerPath);
}
}
}