475 lines
15 KiB
C#
475 lines
15 KiB
C#
using System.IO.MemoryMappedFiles;
|
|
using ZB.MOM.WW.CBDD.Bson;
|
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
|
using ZB.MOM.WW.CBDD.Shared;
|
|
|
|
namespace ZB.MOM.WW.CBDD.Tests;
|
|
|
|
public class CompactionOfflineTests
|
|
{
|
|
/// <summary>
|
|
/// Tests offline compact should preserve logical data equivalence.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_ShouldPreserveLogicalDataEquivalence()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath);
|
|
|
|
var ids = new List<ObjectId>();
|
|
for (var i = 0; i < 160; i++)
|
|
{
|
|
ids.Add(db.Users.Insert(new User { Name = $"user-{i:D4}", Age = i % 31 }));
|
|
}
|
|
|
|
for (var i = 0; i < ids.Count; i += 9)
|
|
{
|
|
if (db.Users.FindById(ids[i]) != null)
|
|
{
|
|
db.Users.Delete(ids[i]).ShouldBeTrue();
|
|
}
|
|
}
|
|
|
|
var updateTargets = db.Users.FindAll(u => u.Age % 4 == 0)
|
|
.Select(u => u.Id)
|
|
.ToList();
|
|
foreach (var id in updateTargets)
|
|
{
|
|
var user = db.Users.FindById(id);
|
|
if (user == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
user.Name += "-updated";
|
|
db.Users.Update(user).ShouldBeTrue();
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var expected = db.Users.FindAll()
|
|
.ToDictionary(u => u.Id, u => (u.Name, u.Age));
|
|
db.SaveChanges();
|
|
|
|
var stats = db.Compact();
|
|
stats.OnlineMode.ShouldBeFalse();
|
|
|
|
var actual = db.Users.FindAll()
|
|
.ToDictionary(u => u.Id, u => (u.Name, u.Age));
|
|
|
|
actual.Count.ShouldBe(expected.Count);
|
|
foreach (var kvp in expected)
|
|
{
|
|
actual.ShouldContainKey(kvp.Key);
|
|
actual[kvp.Key].ShouldBe(kvp.Value);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact should keep index results consistent.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_ShouldKeepIndexResultsConsistent()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath);
|
|
|
|
for (var i = 0; i < 300; i++)
|
|
{
|
|
db.People.Insert(new Person
|
|
{
|
|
Name = $"person-{i:D4}",
|
|
Age = i % 12
|
|
});
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var expectedByAge = db.People.FindAll()
|
|
.GroupBy(p => p.Age)
|
|
.ToDictionary(g => g.Key, g => g.Select(x => x.Name).OrderBy(x => x).ToArray());
|
|
db.SaveChanges();
|
|
|
|
var indexNamesBefore = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
|
|
|
|
var stats = db.Compact(new CompactionOptions
|
|
{
|
|
DefragmentSlottedPages = true,
|
|
NormalizeFreeList = true,
|
|
EnableTailTruncation = true
|
|
});
|
|
stats.PrePageCount.ShouldBeGreaterThanOrEqualTo(stats.PostPageCount);
|
|
|
|
var indexNamesAfter = db.People.GetIndexes().Select(x => x.Name).OrderBy(x => x).ToArray();
|
|
indexNamesAfter.ShouldBe(indexNamesBefore);
|
|
|
|
foreach (var age in expectedByAge.Keys.OrderBy(x => x))
|
|
{
|
|
var actual = db.People.FindAll(p => p.Age == age)
|
|
.Select(x => x.Name)
|
|
.OrderBy(x => x)
|
|
.ToArray();
|
|
|
|
actual.ShouldBe(expectedByAge[age]);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact should rebuild hash index metadata and preserve results.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_ShouldRebuildHashIndexMetadataAndPreserveResults()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath);
|
|
|
|
for (var i = 0; i < 300; i++)
|
|
{
|
|
db.People.Insert(new Person
|
|
{
|
|
Name = $"hash-person-{i:D4}",
|
|
Age = i % 12
|
|
});
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var expectedByAge = db.People.FindAll()
|
|
.GroupBy(p => p.Age)
|
|
.ToDictionary(g => g.Key, g => g.Select(x => x.Name).OrderBy(x => x).ToArray());
|
|
|
|
var metadata = db.Storage.GetCollectionMetadata("people_collection");
|
|
metadata.ShouldNotBeNull();
|
|
|
|
var targetIndex = metadata!.Indexes
|
|
.FirstOrDefault(index => index.PropertyPaths.Any(path => path.Equals("Age", StringComparison.OrdinalIgnoreCase)));
|
|
targetIndex.ShouldNotBeNull();
|
|
|
|
targetIndex!.Type = IndexType.Hash;
|
|
db.Storage.SaveCollectionMetadata(metadata);
|
|
db.SaveChanges();
|
|
|
|
var stats = db.Compact(new CompactionOptions
|
|
{
|
|
DefragmentSlottedPages = true,
|
|
NormalizeFreeList = true,
|
|
EnableTailTruncation = true
|
|
});
|
|
stats.PrePageCount.ShouldBeGreaterThanOrEqualTo(stats.PostPageCount);
|
|
|
|
var reloadedMetadata = db.Storage.GetCollectionMetadata("people_collection");
|
|
reloadedMetadata.ShouldNotBeNull();
|
|
var rebuiltIndex = reloadedMetadata!.Indexes.FirstOrDefault(index => index.Name == targetIndex.Name);
|
|
rebuiltIndex.ShouldNotBeNull();
|
|
rebuiltIndex!.Type.ShouldBe(IndexType.Hash);
|
|
rebuiltIndex.RootPageId.ShouldBeGreaterThan(0u);
|
|
|
|
var runtimeIndex = db.People.GetIndexes().FirstOrDefault(index => index.Name == targetIndex.Name);
|
|
runtimeIndex.ShouldNotBeNull();
|
|
runtimeIndex!.Type.ShouldBe(IndexType.Hash);
|
|
|
|
foreach (var age in expectedByAge.Keys.OrderBy(x => x))
|
|
{
|
|
var actual = db.People.FindAll(p => p.Age == age)
|
|
.Select(x => x.Name)
|
|
.OrderBy(x => x)
|
|
.ToArray();
|
|
|
|
actual.ShouldBe(expectedByAge[age]);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact when tail is reclaimable should reduce file size.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_WhenTailIsReclaimable_ShouldReduceFileSize()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
var ids = new List<ObjectId>();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
|
|
|
for (var i = 0; i < 240; i++)
|
|
{
|
|
var id = db.Users.Insert(new User
|
|
{
|
|
Name = BuildPayload(i, 18_000),
|
|
Age = i
|
|
});
|
|
ids.Add(id);
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
for (var i = ids.Count - 1; i >= 60; i--)
|
|
{
|
|
if (db.Users.FindById(ids[i]) != null)
|
|
{
|
|
db.Users.Delete(ids[i]).ShouldBeTrue();
|
|
}
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var preCompactSize = new FileInfo(dbPath).Length;
|
|
var stats = db.Compact(new CompactionOptions
|
|
{
|
|
EnableTailTruncation = true,
|
|
MinimumRetainedPages = 2
|
|
});
|
|
var postCompactSize = new FileInfo(dbPath).Length;
|
|
|
|
postCompactSize.ShouldBeLessThanOrEqualTo(preCompactSize);
|
|
stats.ReclaimedFileBytes.ShouldBeGreaterThanOrEqualTo(0);
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact with invalid primary root metadata should fail validation.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_WithInvalidPrimaryRootMetadata_ShouldFailValidation()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath);
|
|
for (var i = 0; i < 32; i++)
|
|
{
|
|
db.Users.Insert(new User { Name = $"invalid-primary-{i:D3}", Age = i });
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var metadata = db.Storage.GetCollectionMetadata("users");
|
|
metadata.ShouldNotBeNull();
|
|
metadata!.PrimaryRootPageId = 1; // Metadata page, not an index page.
|
|
db.Storage.SaveCollectionMetadata(metadata);
|
|
|
|
Should.Throw<InvalidDataException>(() => db.Compact())
|
|
.Message.ShouldContain("primary index root page id");
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact with invalid secondary root metadata should fail validation.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_WithInvalidSecondaryRootMetadata_ShouldFailValidation()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath);
|
|
for (var i = 0; i < 48; i++)
|
|
{
|
|
db.People.Insert(new Person { Name = $"invalid-secondary-{i:D3}", Age = i % 10 });
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var metadata = db.Storage.GetCollectionMetadata("people_collection");
|
|
metadata.ShouldNotBeNull();
|
|
metadata!.Indexes.Count.ShouldBeGreaterThan(0);
|
|
metadata.Indexes[0].RootPageId = uint.MaxValue; // Out-of-range page id.
|
|
db.Storage.SaveCollectionMetadata(metadata);
|
|
|
|
Should.Throw<InvalidDataException>(() => db.Compact())
|
|
.Message.ShouldContain("out of range");
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact should report live bytes relocation and throughput telemetry.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_ShouldReportLiveBytesRelocationAndThroughputTelemetry()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
|
|
|
var ids = new List<ObjectId>();
|
|
for (var i = 0; i < 160; i++)
|
|
{
|
|
ids.Add(db.Users.Insert(new User
|
|
{
|
|
Name = BuildPayload(i, 9_000),
|
|
Age = i
|
|
}));
|
|
}
|
|
|
|
for (var i = 0; i < ids.Count; i += 7)
|
|
{
|
|
if (db.Users.FindById(ids[i]) != null)
|
|
{
|
|
db.Users.Delete(ids[i]).ShouldBeTrue();
|
|
}
|
|
}
|
|
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var stats = db.Compact(new CompactionOptions
|
|
{
|
|
DefragmentSlottedPages = true,
|
|
NormalizeFreeList = true,
|
|
EnableTailTruncation = true
|
|
});
|
|
|
|
stats.PreLiveBytes.ShouldBe(Math.Max(0, stats.PreFileSizeBytes - stats.PreFreeBytes));
|
|
stats.PostLiveBytes.ShouldBe(Math.Max(0, stats.PostFileSizeBytes - stats.PostFreeBytes));
|
|
stats.DocumentsRelocated.ShouldBeGreaterThanOrEqualTo(0);
|
|
stats.PagesRelocated.ShouldBeGreaterThanOrEqualTo(0);
|
|
stats.ThroughputBytesPerSecond.ShouldBeGreaterThan(0);
|
|
stats.ThroughputPagesPerSecond.ShouldBeGreaterThanOrEqualTo(0);
|
|
stats.ThroughputDocumentsPerSecond.ShouldBeGreaterThanOrEqualTo(0);
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests offline compact when primary index points to deleted slot should fail validation.
|
|
/// </summary>
|
|
[Fact]
|
|
public void OfflineCompact_WhenPrimaryIndexPointsToDeletedSlot_ShouldFailValidation()
|
|
{
|
|
var dbPath = NewDbPath();
|
|
|
|
try
|
|
{
|
|
using var db = new TestDbContext(dbPath, SmallPageConfig());
|
|
var id = db.Users.Insert(new User { Name = BuildPayload(1, 7_500), Age = 9 });
|
|
db.SaveChanges();
|
|
db.ForceCheckpoint();
|
|
|
|
var metadata = db.Storage.GetCollectionMetadata("users");
|
|
metadata.ShouldNotBeNull();
|
|
metadata!.PrimaryRootPageId.ShouldBeGreaterThan(0u);
|
|
|
|
var primaryIndex = new BTreeIndex(db.Storage, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
|
primaryIndex.TryFind(new IndexKey(id), out var location).ShouldBeTrue();
|
|
|
|
var page = new byte[db.Storage.PageSize];
|
|
db.Storage.ReadPage(location.PageId, null, page);
|
|
|
|
var header = SlottedPageHeader.ReadFrom(page);
|
|
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
|
var slot = SlotEntry.ReadFrom(page.AsSpan(slotOffset, SlotEntry.Size));
|
|
slot.Flags |= SlotFlags.Deleted;
|
|
slot.WriteTo(page.AsSpan(slotOffset, SlotEntry.Size));
|
|
header.WriteTo(page);
|
|
db.Storage.WritePageImmediate(location.PageId, page);
|
|
|
|
var ex = Should.Throw<InvalidDataException>(() => db.Compact(new CompactionOptions
|
|
{
|
|
DefragmentSlottedPages = true,
|
|
NormalizeFreeList = true,
|
|
EnableTailTruncation = true
|
|
}));
|
|
ex.Message.ShouldContain("Compaction validation failed");
|
|
}
|
|
finally
|
|
{
|
|
CleanupFiles(dbPath);
|
|
}
|
|
}
|
|
|
|
private static PageFileConfig SmallPageConfig()
|
|
{
|
|
return new PageFileConfig
|
|
{
|
|
PageSize = 4096,
|
|
InitialFileSize = 1024 * 1024,
|
|
Access = MemoryMappedFileAccess.ReadWrite
|
|
};
|
|
}
|
|
|
|
private static string BuildPayload(int seed, int approxLength)
|
|
{
|
|
var builder = new System.Text.StringBuilder(approxLength + 256);
|
|
var i = 0;
|
|
while (builder.Length < approxLength)
|
|
{
|
|
builder.Append("compact-tail-");
|
|
builder.Append(seed.ToString("D4"));
|
|
builder.Append('-');
|
|
builder.Append(i.ToString("D6"));
|
|
builder.Append('|');
|
|
i++;
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
private static string NewDbPath()
|
|
=> Path.Combine(Path.GetTempPath(), $"compaction_offline_{Guid.NewGuid():N}.db");
|
|
|
|
private static void CleanupFiles(string dbPath)
|
|
{
|
|
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
|
var markerPath = $"{dbPath}.compact.state";
|
|
var tempPath = $"{dbPath}.compact.tmp";
|
|
var backupPath = $"{dbPath}.compact.bak";
|
|
if (File.Exists(dbPath)) File.Delete(dbPath);
|
|
if (File.Exists(walPath)) File.Delete(walPath);
|
|
if (File.Exists(markerPath)) File.Delete(markerPath);
|
|
if (File.Exists(tempPath)) File.Delete(tempPath);
|
|
if (File.Exists(backupPath)) File.Delete(backupPath);
|
|
}
|
|
}
|