Reformat / cleanup
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
using System.Diagnostics;
|
||||
using System.IO.Compression;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Serilog.Context;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
@@ -10,47 +11,47 @@ namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
||||
|
||||
internal static class DatabaseSizeBenchmark
|
||||
{
|
||||
private const int BatchSize = 50_000;
|
||||
private const int ProgressInterval = 1_000_000;
|
||||
private static readonly int[] TargetCounts = [10_000, 1_000_000, 10_000_000];
|
||||
|
||||
private static readonly CompressionOptions CompressedBrotliFast = new()
|
||||
{
|
||||
EnableCompression = true,
|
||||
MinSizeBytes = 256,
|
||||
MinSavingsPercent = 0,
|
||||
Codec = CompressionCodec.Brotli,
|
||||
Level = System.IO.Compression.CompressionLevel.Fastest
|
||||
Level = CompressionLevel.Fastest
|
||||
};
|
||||
|
||||
private static readonly Scenario[] Scenarios =
|
||||
[
|
||||
// Separate compression set (no compaction)
|
||||
new(
|
||||
Set: "compression",
|
||||
Name: "CompressionOnly-Uncompressed",
|
||||
CompressionOptions: CompressionOptions.Default,
|
||||
RunCompaction: false),
|
||||
"compression",
|
||||
"CompressionOnly-Uncompressed",
|
||||
CompressionOptions.Default,
|
||||
false),
|
||||
new(
|
||||
Set: "compression",
|
||||
Name: "CompressionOnly-Compressed-BrotliFast",
|
||||
CompressionOptions: CompressedBrotliFast,
|
||||
RunCompaction: false),
|
||||
"compression",
|
||||
"CompressionOnly-Compressed-BrotliFast",
|
||||
CompressedBrotliFast,
|
||||
false),
|
||||
// Separate compaction set (compaction enabled)
|
||||
new(
|
||||
Set: "compaction",
|
||||
Name: "Compaction-Uncompressed",
|
||||
CompressionOptions: CompressionOptions.Default,
|
||||
RunCompaction: true),
|
||||
"compaction",
|
||||
"Compaction-Uncompressed",
|
||||
CompressionOptions.Default,
|
||||
true),
|
||||
new(
|
||||
Set: "compaction",
|
||||
Name: "Compaction-Compressed-BrotliFast",
|
||||
CompressionOptions: CompressedBrotliFast,
|
||||
RunCompaction: true)
|
||||
"compaction",
|
||||
"Compaction-Compressed-BrotliFast",
|
||||
CompressedBrotliFast,
|
||||
true)
|
||||
];
|
||||
|
||||
private const int BatchSize = 50_000;
|
||||
private const int ProgressInterval = 1_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// Tests run.
|
||||
/// Tests run.
|
||||
/// </summary>
|
||||
/// <param name="logger">Logger for benchmark progress and results.</param>
|
||||
public static void Run(ILogger logger)
|
||||
@@ -62,109 +63,101 @@ internal static class DatabaseSizeBenchmark
|
||||
logger.LogInformation("Scenarios: {Scenarios}", string.Join(", ", Scenarios.Select(x => $"{x.Set}:{x.Name}")));
|
||||
logger.LogInformation("Batch size: {BatchSize:N0}", BatchSize);
|
||||
|
||||
foreach (var targetCount in TargetCounts)
|
||||
foreach (int targetCount in TargetCounts)
|
||||
foreach (var scenario in Scenarios)
|
||||
{
|
||||
foreach (var scenario in Scenarios)
|
||||
string dbPath = Path.Combine(Path.GetTempPath(),
|
||||
$"cbdd_size_{scenario.Name}_{targetCount}_{Guid.NewGuid():N}.db");
|
||||
string walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||
using var _ = LogContext.PushProperty("TargetCount", targetCount);
|
||||
using var __ = LogContext.PushProperty("Scenario", scenario.Name);
|
||||
using var ___ = LogContext.PushProperty("ScenarioSet", scenario.Set);
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting {Set} scenario {Scenario} for target {TargetCount:N0} docs",
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
targetCount);
|
||||
|
||||
var insertStopwatch = Stopwatch.StartNew();
|
||||
CompressionStats compressionStats = default;
|
||||
CompactionStats compactionStats = new();
|
||||
long preCompactDbBytes;
|
||||
long preCompactWalBytes;
|
||||
long postCompactDbBytes;
|
||||
long postCompactWalBytes;
|
||||
|
||||
using (var storage = new StorageEngine(dbPath, PageFileConfig.Default, scenario.CompressionOptions))
|
||||
using (var transactionHolder = new BenchmarkTransactionHolder(storage))
|
||||
{
|
||||
var dbPath = Path.Combine(Path.GetTempPath(), $"cbdd_size_{scenario.Name}_{targetCount}_{Guid.NewGuid():N}.db");
|
||||
var walPath = Path.ChangeExtension(dbPath, ".wal");
|
||||
using var _ = LogContext.PushProperty("TargetCount", targetCount);
|
||||
using var __ = LogContext.PushProperty("Scenario", scenario.Name);
|
||||
using var ___ = LogContext.PushProperty("ScenarioSet", scenario.Set);
|
||||
var collection = new DocumentCollection<SizeBenchmarkDocument>(
|
||||
storage,
|
||||
transactionHolder,
|
||||
new SizeBenchmarkDocumentMapper());
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting {Set} scenario {Scenario} for target {TargetCount:N0} docs",
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
targetCount);
|
||||
|
||||
var insertStopwatch = Stopwatch.StartNew();
|
||||
CompressionStats compressionStats = default;
|
||||
CompactionStats compactionStats = new();
|
||||
long preCompactDbBytes;
|
||||
long preCompactWalBytes;
|
||||
long postCompactDbBytes;
|
||||
long postCompactWalBytes;
|
||||
|
||||
using (var storage = new StorageEngine(dbPath, PageFileConfig.Default, scenario.CompressionOptions))
|
||||
using (var transactionHolder = new BenchmarkTransactionHolder(storage))
|
||||
var inserted = 0;
|
||||
while (inserted < targetCount)
|
||||
{
|
||||
var collection = new DocumentCollection<SizeBenchmarkDocument>(
|
||||
storage,
|
||||
transactionHolder,
|
||||
new SizeBenchmarkDocumentMapper());
|
||||
int currentBatchSize = Math.Min(BatchSize, targetCount - inserted);
|
||||
var documents = new SizeBenchmarkDocument[currentBatchSize];
|
||||
int baseValue = inserted;
|
||||
|
||||
var inserted = 0;
|
||||
while (inserted < targetCount)
|
||||
{
|
||||
var currentBatchSize = Math.Min(BatchSize, targetCount - inserted);
|
||||
var documents = new SizeBenchmarkDocument[currentBatchSize];
|
||||
var baseValue = inserted;
|
||||
for (var i = 0; i < currentBatchSize; i++) documents[i] = CreateDocument(baseValue + i);
|
||||
|
||||
for (var i = 0; i < currentBatchSize; i++)
|
||||
{
|
||||
documents[i] = CreateDocument(baseValue + i);
|
||||
}
|
||||
collection.InsertBulk(documents);
|
||||
transactionHolder.CommitAndReset();
|
||||
|
||||
collection.InsertBulk(documents);
|
||||
transactionHolder.CommitAndReset();
|
||||
|
||||
inserted += currentBatchSize;
|
||||
if (inserted == targetCount || inserted % ProgressInterval == 0)
|
||||
{
|
||||
logger.LogInformation("Inserted {Inserted:N0}/{TargetCount:N0}", inserted, targetCount);
|
||||
}
|
||||
}
|
||||
|
||||
insertStopwatch.Stop();
|
||||
preCompactDbBytes = File.Exists(dbPath) ? new FileInfo(dbPath).Length : 0;
|
||||
preCompactWalBytes = File.Exists(walPath) ? new FileInfo(walPath).Length : 0;
|
||||
|
||||
if (scenario.RunCompaction)
|
||||
{
|
||||
compactionStats = storage.Compact(new CompactionOptions
|
||||
{
|
||||
EnableTailTruncation = true,
|
||||
DefragmentSlottedPages = true,
|
||||
NormalizeFreeList = true
|
||||
});
|
||||
}
|
||||
|
||||
postCompactDbBytes = File.Exists(dbPath) ? new FileInfo(dbPath).Length : 0;
|
||||
postCompactWalBytes = File.Exists(walPath) ? new FileInfo(walPath).Length : 0;
|
||||
compressionStats = storage.GetCompressionStats();
|
||||
inserted += currentBatchSize;
|
||||
if (inserted == targetCount || inserted % ProgressInterval == 0)
|
||||
logger.LogInformation("Inserted {Inserted:N0}/{TargetCount:N0}", inserted, targetCount);
|
||||
}
|
||||
|
||||
var result = new SizeResult(
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
scenario.RunCompaction,
|
||||
targetCount,
|
||||
insertStopwatch.Elapsed,
|
||||
preCompactDbBytes,
|
||||
preCompactWalBytes,
|
||||
postCompactDbBytes,
|
||||
postCompactWalBytes,
|
||||
compactionStats,
|
||||
compressionStats);
|
||||
results.Add(result);
|
||||
insertStopwatch.Stop();
|
||||
preCompactDbBytes = File.Exists(dbPath) ? new FileInfo(dbPath).Length : 0;
|
||||
preCompactWalBytes = File.Exists(walPath) ? new FileInfo(walPath).Length : 0;
|
||||
|
||||
logger.LogInformation(
|
||||
"Completed {Set}:{Scenario} {TargetCount:N0} docs in {Elapsed}. pre={PreTotal}, post={PostTotal}, shrink={Shrink}, compactApplied={CompactionApplied}, compactReclaim={CompactReclaim}, compRatio={CompRatio}",
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
targetCount,
|
||||
insertStopwatch.Elapsed,
|
||||
FormatBytes(result.PreCompactTotalBytes),
|
||||
FormatBytes(result.PostCompactTotalBytes),
|
||||
FormatBytes(result.ShrinkBytes),
|
||||
scenario.RunCompaction,
|
||||
FormatBytes(result.CompactionStats.ReclaimedFileBytes),
|
||||
result.CompressionRatioText);
|
||||
if (scenario.RunCompaction)
|
||||
compactionStats = storage.Compact(new CompactionOptions
|
||||
{
|
||||
EnableTailTruncation = true,
|
||||
DefragmentSlottedPages = true,
|
||||
NormalizeFreeList = true
|
||||
});
|
||||
|
||||
TryDelete(dbPath);
|
||||
TryDelete(walPath);
|
||||
postCompactDbBytes = File.Exists(dbPath) ? new FileInfo(dbPath).Length : 0;
|
||||
postCompactWalBytes = File.Exists(walPath) ? new FileInfo(walPath).Length : 0;
|
||||
compressionStats = storage.GetCompressionStats();
|
||||
}
|
||||
|
||||
var result = new SizeResult(
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
scenario.RunCompaction,
|
||||
targetCount,
|
||||
insertStopwatch.Elapsed,
|
||||
preCompactDbBytes,
|
||||
preCompactWalBytes,
|
||||
postCompactDbBytes,
|
||||
postCompactWalBytes,
|
||||
compactionStats,
|
||||
compressionStats);
|
||||
results.Add(result);
|
||||
|
||||
logger.LogInformation(
|
||||
"Completed {Set}:{Scenario} {TargetCount:N0} docs in {Elapsed}. pre={PreTotal}, post={PostTotal}, shrink={Shrink}, compactApplied={CompactionApplied}, compactReclaim={CompactReclaim}, compRatio={CompRatio}",
|
||||
scenario.Set,
|
||||
scenario.Name,
|
||||
targetCount,
|
||||
insertStopwatch.Elapsed,
|
||||
FormatBytes(result.PreCompactTotalBytes),
|
||||
FormatBytes(result.PostCompactTotalBytes),
|
||||
FormatBytes(result.ShrinkBytes),
|
||||
scenario.RunCompaction,
|
||||
FormatBytes(result.CompactionStats.ReclaimedFileBytes),
|
||||
result.CompressionRatioText);
|
||||
|
||||
TryDelete(dbPath);
|
||||
TryDelete(walPath);
|
||||
}
|
||||
|
||||
logger.LogInformation("=== Size Benchmark Summary ===");
|
||||
@@ -172,7 +165,6 @@ internal static class DatabaseSizeBenchmark
|
||||
.OrderBy(x => x.Set)
|
||||
.ThenBy(x => x.TargetCount)
|
||||
.ThenBy(x => x.Scenario))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"{Set,-11} | {Scenario,-38} | {Count,12:N0} docs | insert={Elapsed,12} | pre={Pre,12} | post={Post,12} | shrink={Shrink,12} | compact={CompactBytes,12} | ratio={Ratio}",
|
||||
result.Set,
|
||||
@@ -184,7 +176,6 @@ internal static class DatabaseSizeBenchmark
|
||||
FormatBytes(result.ShrinkBytes),
|
||||
FormatBytes(result.CompactionStats.ReclaimedFileBytes),
|
||||
result.CompressionRatioText);
|
||||
}
|
||||
|
||||
WriteSummaryCsv(results, logger);
|
||||
}
|
||||
@@ -201,10 +192,7 @@ internal static class DatabaseSizeBenchmark
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
if (File.Exists(path)) File.Delete(path);
|
||||
}
|
||||
|
||||
private static string FormatBytes(long bytes)
|
||||
@@ -224,9 +212,9 @@ internal static class DatabaseSizeBenchmark
|
||||
|
||||
private static void WriteSummaryCsv(IEnumerable<SizeResult> results, ILogger logger)
|
||||
{
|
||||
var outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts", "results");
|
||||
string outputDirectory = Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Artifacts", "results");
|
||||
Directory.CreateDirectory(outputDirectory);
|
||||
var outputPath = Path.Combine(outputDirectory, "DatabaseSizeBenchmark-results.csv");
|
||||
string outputPath = Path.Combine(outputDirectory, "DatabaseSizeBenchmark-results.csv");
|
||||
|
||||
var lines = new List<string>
|
||||
{
|
||||
@@ -234,7 +222,6 @@ internal static class DatabaseSizeBenchmark
|
||||
};
|
||||
|
||||
foreach (var result in results.OrderBy(x => x.Set).ThenBy(x => x.TargetCount).ThenBy(x => x.Scenario))
|
||||
{
|
||||
lines.Add(string.Join(",",
|
||||
result.Set,
|
||||
result.Scenario,
|
||||
@@ -246,7 +233,6 @@ internal static class DatabaseSizeBenchmark
|
||||
result.ShrinkBytes.ToString(),
|
||||
result.CompactionStats.ReclaimedFileBytes.ToString(),
|
||||
result.CompressionRatioText));
|
||||
}
|
||||
|
||||
File.WriteAllLines(outputPath, lines);
|
||||
logger.LogInformation("Database size summary CSV written to {OutputPath}", outputPath);
|
||||
@@ -268,20 +254,22 @@ internal static class DatabaseSizeBenchmark
|
||||
CompressionStats CompressionStats)
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the pre compact total bytes.
|
||||
/// Gets or sets the pre compact total bytes.
|
||||
/// </summary>
|
||||
public long PreCompactTotalBytes => PreCompactDbBytes + PreCompactWalBytes;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the post compact total bytes.
|
||||
/// Gets or sets the post compact total bytes.
|
||||
/// </summary>
|
||||
public long PostCompactTotalBytes => PostCompactDbBytes + PostCompactWalBytes;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the shrink bytes.
|
||||
/// Gets or sets the shrink bytes.
|
||||
/// </summary>
|
||||
public long ShrinkBytes => PreCompactTotalBytes - PostCompactTotalBytes;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the compression ratio text.
|
||||
/// Gets or sets the compression ratio text.
|
||||
/// </summary>
|
||||
public string CompressionRatioText =>
|
||||
CompressionStats.BytesAfterCompression > 0
|
||||
@@ -292,15 +280,17 @@ internal static class DatabaseSizeBenchmark
|
||||
private sealed class SizeBenchmarkDocument
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the id.
|
||||
/// Gets or sets the id.
|
||||
/// </summary>
|
||||
public ObjectId Id { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the value.
|
||||
/// Gets or sets the value.
|
||||
/// </summary>
|
||||
public int Value { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the name.
|
||||
/// Gets or sets the name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
}
|
||||
@@ -311,15 +301,21 @@ internal static class DatabaseSizeBenchmark
|
||||
public override string CollectionName => "size_documents";
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ObjectId GetId(SizeBenchmarkDocument entity) => entity.Id;
|
||||
public override ObjectId GetId(SizeBenchmarkDocument entity)
|
||||
{
|
||||
return entity.Id;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void SetId(SizeBenchmarkDocument entity, ObjectId id) => entity.Id = id;
|
||||
public override void SetId(SizeBenchmarkDocument entity, ObjectId id)
|
||||
{
|
||||
entity.Id = id;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int Serialize(SizeBenchmarkDocument entity, BsonSpanWriter writer)
|
||||
{
|
||||
var sizePos = writer.BeginDocument();
|
||||
int sizePos = writer.BeginDocument();
|
||||
writer.WriteObjectId("_id", entity.Id);
|
||||
writer.WriteInt32("value", entity.Value);
|
||||
writer.WriteString("name", entity.Name);
|
||||
@@ -336,12 +332,9 @@ internal static class DatabaseSizeBenchmark
|
||||
while (reader.Remaining > 0)
|
||||
{
|
||||
var bsonType = reader.ReadBsonType();
|
||||
if (bsonType == BsonType.EndOfDocument)
|
||||
{
|
||||
break;
|
||||
}
|
||||
if (bsonType == BsonType.EndOfDocument) break;
|
||||
|
||||
var name = reader.ReadElementHeader();
|
||||
string name = reader.ReadElementHeader();
|
||||
switch (name)
|
||||
{
|
||||
case "_id":
|
||||
@@ -362,4 +355,4 @@ internal static class DatabaseSizeBenchmark
|
||||
return document;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user