Initialize CBDD solution and add a .NET-focused gitignore for generated artifacts.
This commit is contained in:
Joseph Doherty
379
tests/CBDD.Tests/CompactionOfflineTests.cs
Normal file
379
tests/CBDD.Tests/CompactionOfflineTests.cs
Normal file
@@ -0,0 +1,379 @@
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Shared;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Tests;
|
||||
|
||||
public class CompactionOfflineTests
|
||||
{
|
||||
[Fact]
|
||||
public void OfflineCompact_ShouldPreserveLogicalDataEquivalence()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
|
||||
var ids = new List<ObjectId>();
|
||||
for (var i = 0; i < 160; i++)
|
||||
{
|
||||
ids.Add(db.Users.Insert(new User { Name = $"user-{i:D4}", Age = i % 31 }));
|
||||
}
|
||||
|
||||
for (var i = 0; i < ids.Count; i += 9)
|
||||
{
|
||||
if (db.Users.FindById(ids[i]) != null)
|
||||
{
|
||||
db.Users.Delete(ids[i]).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
var updateTargets = db.Users.FindAll(u => u.Age % 4 == 0)
|
||||
.Select(u => u.Id)
|
||||
.ToList();
|
||||
foreach (var id in updateTargets)
|
||||
{
|
||||
var user = db.Users.FindById(id);
|
||||
if (user == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
user.Name += "-updated";
|
||||
db.Users.Update(user).ShouldBeTrue();
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var expected = db.Users.FindAll()
|
||||
.ToDictionary(u => u.Id, u => (u.Name, u.Age));
|
||||
db.SaveChanges();
|
||||
|
||||
var stats = db.Compact();
|
||||
stats.OnlineMode.ShouldBeFalse();
|
||||
|
||||
var actual = db.Users.FindAll()
|
||||
.ToDictionary(u => u.Id, u => (u.Name, u.Age));
|
||||
|
||||
actual.Count.ShouldBe(expected.Count);
|
||||
foreach (var kvp in expected)
|
||||
{
|
||||
actual.ShouldContainKey(kvp.Key);
|
||||
actual[kvp.Key].ShouldBe(kvp.Value);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_ShouldKeepIndexResultsConsistent()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
|
||||
for (var i = 0; i < 300; i++)
|
||||
{
|
||||
db.People.Insert(new Person
|
||||
{
|
||||
Name = $"person-{i:D4}",
|
||||
Age = i % 12
|
||||
});
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var expectedByAge = db.People.FindAll()
|
||||
.GroupBy(p => p.Age)
|
||||
.ToDictionary(g => g.Key, g => g.Select(x => x.Name).OrderBy(x => x).ToArray());
|
||||
db.SaveChanges();
|
||||
|
||||
var indexNamesBefore = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
|
||||
|
||||
var stats = db.Compact(new CompactionOptions
|
||||
{
|
||||
DefragmentSlottedPages = true,
|
||||
NormalizeFreeList = true,
|
||||
EnableTailTruncation = true
|
||||
});
|
||||
stats.PrePageCount.ShouldBeGreaterThanOrEqualTo(stats.PostPageCount);
|
||||
|
||||
var indexNamesAfter = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
|
||||
indexNamesAfter.ShouldBe(indexNamesBefore);
|
||||
|
||||
foreach (var age in expectedByAge.Keys.OrderBy(x => x))
|
||||
{
|
||||
var actual = db.People.FindAll(p => p.Age == age)
|
||||
.Select(x => x.Name)
|
||||
.OrderBy(x => x)
|
||||
.ToArray();
|
||||
|
||||
actual.ShouldBe(expectedByAge[age]);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_WhenTailIsReclaimable_ShouldReduceFileSize()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
var ids = new List<ObjectId>();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
||||
|
||||
for (var i = 0; i < 240; i++)
|
||||
{
|
||||
var id = db.Users.Insert(new User
|
||||
{
|
||||
Name = BuildPayload(i, 18_000),
|
||||
Age = i
|
||||
});
|
||||
ids.Add(id);
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
for (var i = ids.Count - 1; i >= 60; i--)
|
||||
{
|
||||
if (db.Users.FindById(ids[i]) != null)
|
||||
{
|
||||
db.Users.Delete(ids[i]).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var preCompactSize = new FileInfo(dbPath).Length;
|
||||
var stats = db.Compact(new CompactionOptions
|
||||
{
|
||||
EnableTailTruncation = true,
|
||||
MinimumRetainedPages = 2
|
||||
});
|
||||
var postCompactSize = new FileInfo(dbPath).Length;
|
||||
|
||||
postCompactSize.ShouldBeLessThanOrEqualTo(preCompactSize);
|
||||
stats.ReclaimedFileBytes.ShouldBeGreaterThanOrEqualTo(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_WithInvalidPrimaryRootMetadata_ShouldFailValidation()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
for (var i = 0; i < 32; i++)
|
||||
{
|
||||
db.Users.Insert(new User { Name = $"invalid-primary-{i:D3}", Age = i });
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var metadata = db.Storage.GetCollectionMetadata("users");
|
||||
metadata.ShouldNotBeNull();
|
||||
metadata!.PrimaryRootPageId = 1; // Metadata page, not an index page.
|
||||
db.Storage.SaveCollectionMetadata(metadata);
|
||||
|
||||
Should.Throw<InvalidDataException>(() => db.Compact())
|
||||
.Message.ShouldContain("primary index root page id");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_WithInvalidSecondaryRootMetadata_ShouldFailValidation()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath);
|
||||
for (var i = 0; i < 48; i++)
|
||||
{
|
||||
db.People.Insert(new Person { Name = $"invalid-secondary-{i:D3}", Age = i % 10 });
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var metadata = db.Storage.GetCollectionMetadata("people_collection");
|
||||
metadata.ShouldNotBeNull();
|
||||
metadata!.Indexes.Count.ShouldBeGreaterThan(0);
|
||||
metadata.Indexes[0].RootPageId = uint.MaxValue; // Out-of-range page id.
|
||||
db.Storage.SaveCollectionMetadata(metadata);
|
||||
|
||||
Should.Throw<InvalidDataException>(() => db.Compact())
|
||||
.Message.ShouldContain("out of range");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_ShouldReportLiveBytesRelocationAndThroughputTelemetry()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
||||
|
||||
var ids = new List<ObjectId>();
|
||||
for (var i = 0; i < 160; i++)
|
||||
{
|
||||
ids.Add(db.Users.Insert(new User
|
||||
{
|
||||
Name = BuildPayload(i, 9_000),
|
||||
Age = i
|
||||
}));
|
||||
}
|
||||
|
||||
for (var i = 0; i < ids.Count; i += 7)
|
||||
{
|
||||
if (db.Users.FindById(ids[i]) != null)
|
||||
{
|
||||
db.Users.Delete(ids[i]).ShouldBeTrue();
|
||||
}
|
||||
}
|
||||
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var stats = db.Compact(new CompactionOptions
|
||||
{
|
||||
DefragmentSlottedPages = true,
|
||||
NormalizeFreeList = true,
|
||||
EnableTailTruncation = true
|
||||
});
|
||||
|
||||
stats.PreLiveBytes.ShouldBe(Math.Max(0, stats.PreFileSizeBytes - stats.PreFreeBytes));
|
||||
stats.PostLiveBytes.ShouldBe(Math.Max(0, stats.PostFileSizeBytes - stats.PostFreeBytes));
|
||||
stats.DocumentsRelocated.ShouldBeGreaterThanOrEqualTo(0);
|
||||
stats.PagesRelocated.ShouldBeGreaterThanOrEqualTo(0);
|
||||
stats.ThroughputBytesPerSecond.ShouldBeGreaterThan(0);
|
||||
stats.ThroughputPagesPerSecond.ShouldBeGreaterThanOrEqualTo(0);
|
||||
stats.ThroughputDocumentsPerSecond.ShouldBeGreaterThanOrEqualTo(0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OfflineCompact_WhenPrimaryIndexPointsToDeletedSlot_ShouldFailValidation()
|
||||
{
|
||||
var dbPath = NewDbPath();
|
||||
|
||||
try
|
||||
{
|
||||
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
||||
var id = db.Users.Insert(new User { Name = BuildPayload(1, 7_500), Age = 9 });
|
||||
db.SaveChanges();
|
||||
db.ForceCheckpoint();
|
||||
|
||||
var metadata = db.Storage.GetCollectionMetadata("users");
|
||||
metadata.ShouldNotBeNull();
|
||||
metadata!.PrimaryRootPageId.ShouldBeGreaterThan(0u);
|
||||
|
||||
var primaryIndex = new BTreeIndex(db.Storage, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
||||
primaryIndex.TryFind(new IndexKey(id), out var location).ShouldBeTrue();
|
||||
|
||||
var page = new byte[db.Storage.PageSize];
|
||||
db.Storage.ReadPage(location.PageId, null, page);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(page);
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(page.AsSpan(slotOffset, SlotEntry.Size));
|
||||
slot.Flags |= SlotFlags.Deleted;
|
||||
slot.WriteTo(page.AsSpan(slotOffset, SlotEntry.Size));
|
||||
header.WriteTo(page);
|
||||
db.Storage.WritePageImmediate(location.PageId, page);
|
||||
|
||||
var ex = Should.Throw<InvalidDataException>(() => db.Compact(new CompactionOptions
|
||||
{
|
||||
DefragmentSlottedPages = true,
|
||||
NormalizeFreeList = true,
|
||||
EnableTailTruncation = true
|
||||
}));
|
||||
ex.Message.ShouldContain("Compaction validation failed");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupFiles(dbPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static PageFileConfig SmallPageConfig()
|
||||
{
|
||||
return new PageFileConfig
|
||||
{
|
||||
PageSize = 4096,
|
||||
InitialFileSize = 1024 * 1024,
|
||||
Access = MemoryMappedFileAccess.ReadWrite
|
||||
};
|
||||
}
|
||||
|
||||
private static string BuildPayload(int seed, int approxLength)
|
||||
{
|
||||
var builder = new System.Text.StringBuilder(approxLength + 256);
|
||||
var i = 0;
|
||||
while (builder.Length < approxLength)
|
||||
{
|
||||
builder.Append("compact-tail-");
|
||||
builder.Append(seed.ToString("D4"));
|
||||
builder.Append('-');
|
||||
builder.Append(i.ToString("D6"));
|
||||
builder.Append('|');
|
||||
i++;
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static string NewDbPath()
|
||||
=> Path.Combine(Path.GetTempPath(), $"compaction_offline_{Guid.NewGuid():N}.db");
|
||||
|
||||
private static void CleanupFiles(string dbPath)
|
||||
{
|
||||
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||
var markerPath = $"{dbPath}.compact.state";
|
||||
var tempPath = $"{dbPath}.compact.tmp";
|
||||
var backupPath = $"{dbPath}.compact.bak";
|
||||
if (File.Exists(dbPath)) File.Delete(dbPath);
|
||||
if (File.Exists(walPath)) File.Delete(walPath);
|
||||
if (File.Exists(markerPath)) File.Delete(markerPath);
|
||||
if (File.Exists(tempPath)) File.Delete(tempPath);
|
||||
if (File.Exists(backupPath)) File.Delete(backupPath);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user