Initialize CBDD solution and add a .NET-focused gitignore for generated artifacts.
This commit is contained in:
Joseph Doherty
448
src/CBDD.Core/Storage/StorageEngine.Diagnostics.cs
Normal file
448
src/CBDD.Core/Storage/StorageEngine.Diagnostics.cs
Normal file
@@ -0,0 +1,448 @@
|
||||
using System.Buffers.Binary;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Aggregated page counts grouped by page type.
|
||||
/// </summary>
|
||||
public sealed class PageTypeUsageEntry
|
||||
{
|
||||
public PageType PageType { get; init; }
|
||||
public int PageCount { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-collection page usage summary.
|
||||
/// </summary>
|
||||
public sealed class CollectionPageUsageEntry
|
||||
{
|
||||
public string CollectionName { get; init; } = string.Empty;
|
||||
public int TotalDistinctPages { get; init; }
|
||||
public int DataPages { get; init; }
|
||||
public int OverflowPages { get; init; }
|
||||
public int IndexPages { get; init; }
|
||||
public int OtherPages { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-collection compression ratio summary.
|
||||
/// </summary>
|
||||
public sealed class CollectionCompressionRatioEntry
|
||||
{
|
||||
public string CollectionName { get; init; } = string.Empty;
|
||||
public long DocumentCount { get; init; }
|
||||
public long CompressedDocumentCount { get; init; }
|
||||
public long BytesBeforeCompression { get; init; }
|
||||
public long BytesAfterCompression { get; init; }
|
||||
public double CompressionRatio => BytesAfterCompression <= 0 ? 1.0 : (double)BytesBeforeCompression / BytesAfterCompression;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summary of free-list and reclaimable tail information.
|
||||
/// </summary>
|
||||
public sealed class FreeListSummary
|
||||
{
|
||||
public uint PageCount { get; init; }
|
||||
public int FreePageCount { get; init; }
|
||||
public long FreeBytes { get; init; }
|
||||
public double FragmentationPercent { get; init; }
|
||||
public uint TailReclaimablePages { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single page entry in fragmentation reporting.
|
||||
/// </summary>
|
||||
public sealed class FragmentationPageEntry
|
||||
{
|
||||
public uint PageId { get; init; }
|
||||
public PageType PageType { get; init; }
|
||||
public bool IsFreePage { get; init; }
|
||||
public int FreeBytes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detailed fragmentation map and totals.
|
||||
/// </summary>
|
||||
public sealed class FragmentationMapReport
|
||||
{
|
||||
public IReadOnlyList<FragmentationPageEntry> Pages { get; init; } = Array.Empty<FragmentationPageEntry>();
|
||||
public long TotalFreeBytes { get; init; }
|
||||
public double FragmentationPercent { get; init; }
|
||||
public uint TailReclaimablePages { get; init; }
|
||||
}
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets page usage grouped by page type.
|
||||
/// </summary>
|
||||
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
||||
{
|
||||
var pageCount = _pageFile.NextPageId;
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var counts = new Dictionary<PageType, int>();
|
||||
|
||||
for (uint pageId = 0; pageId < pageCount; pageId++)
|
||||
{
|
||||
_pageFile.ReadPage(pageId, buffer);
|
||||
var pageType = PageHeader.ReadFrom(buffer).PageType;
|
||||
counts[pageType] = counts.TryGetValue(pageType, out var count) ? count + 1 : 1;
|
||||
}
|
||||
|
||||
return counts
|
||||
.OrderBy(x => (byte)x.Key)
|
||||
.Select(x => new PageTypeUsageEntry
|
||||
{
|
||||
PageType = x.Key,
|
||||
PageCount = x.Value
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection page usage by resolving primary-index locations and related index roots.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
||||
{
|
||||
var metadataEntries = GetAllCollectionMetadata();
|
||||
var results = new List<CollectionPageUsageEntry>(metadataEntries.Count);
|
||||
|
||||
foreach (var metadata in metadataEntries)
|
||||
{
|
||||
var pageIds = new HashSet<uint>();
|
||||
|
||||
if (metadata.PrimaryRootPageId != 0)
|
||||
pageIds.Add(metadata.PrimaryRootPageId);
|
||||
if (metadata.SchemaRootPageId != 0)
|
||||
pageIds.Add(metadata.SchemaRootPageId);
|
||||
|
||||
foreach (var indexMetadata in metadata.Indexes)
|
||||
{
|
||||
if (indexMetadata.RootPageId != 0)
|
||||
pageIds.Add(indexMetadata.RootPageId);
|
||||
}
|
||||
|
||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||
{
|
||||
pageIds.Add(location.PageId);
|
||||
if (TryReadFirstOverflowPage(location, out var firstOverflowPage))
|
||||
{
|
||||
AddOverflowChainPages(pageIds, firstOverflowPage);
|
||||
}
|
||||
}
|
||||
|
||||
int data = 0;
|
||||
int overflow = 0;
|
||||
int indexPages = 0;
|
||||
int other = 0;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
foreach (var pageId in pageIds)
|
||||
{
|
||||
if (pageId >= _pageFile.NextPageId)
|
||||
continue;
|
||||
|
||||
_pageFile.ReadPage(pageId, pageBuffer);
|
||||
var pageType = PageHeader.ReadFrom(pageBuffer).PageType;
|
||||
|
||||
if (pageType == PageType.Data)
|
||||
{
|
||||
data++;
|
||||
}
|
||||
else if (pageType == PageType.Overflow)
|
||||
{
|
||||
overflow++;
|
||||
}
|
||||
else if (pageType == PageType.Index || pageType == PageType.Vector || pageType == PageType.Spatial)
|
||||
{
|
||||
indexPages++;
|
||||
}
|
||||
else
|
||||
{
|
||||
other++;
|
||||
}
|
||||
}
|
||||
|
||||
results.Add(new CollectionPageUsageEntry
|
||||
{
|
||||
CollectionName = metadata.Name,
|
||||
TotalDistinctPages = pageIds.Count,
|
||||
DataPages = data,
|
||||
OverflowPages = overflow,
|
||||
IndexPages = indexPages,
|
||||
OtherPages = other
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection logical-vs-stored compression ratios.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
||||
{
|
||||
var metadataEntries = GetAllCollectionMetadata();
|
||||
var results = new List<CollectionCompressionRatioEntry>(metadataEntries.Count);
|
||||
|
||||
foreach (var metadata in metadataEntries)
|
||||
{
|
||||
long docs = 0;
|
||||
long compressedDocs = 0;
|
||||
long bytesBefore = 0;
|
||||
long bytesAfter = 0;
|
||||
|
||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||
{
|
||||
if (!TryReadSlotPayloadStats(location, out var isCompressed, out var originalBytes, out var storedBytes))
|
||||
continue;
|
||||
|
||||
docs++;
|
||||
if (isCompressed)
|
||||
compressedDocs++;
|
||||
|
||||
bytesBefore += originalBytes;
|
||||
bytesAfter += storedBytes;
|
||||
}
|
||||
|
||||
results.Add(new CollectionCompressionRatioEntry
|
||||
{
|
||||
CollectionName = metadata.Name,
|
||||
DocumentCount = docs,
|
||||
CompressedDocumentCount = compressedDocs,
|
||||
BytesBeforeCompression = bytesBefore,
|
||||
BytesAfterCompression = bytesAfter
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets free-list summary for diagnostics.
|
||||
/// </summary>
|
||||
public FreeListSummary GetFreeListSummary()
|
||||
{
|
||||
var snapshot = CaptureCompactionSnapshot();
|
||||
return new FreeListSummary
|
||||
{
|
||||
PageCount = snapshot.PageCount,
|
||||
FreePageCount = snapshot.FreePageCount,
|
||||
FreeBytes = snapshot.TotalFreeBytes,
|
||||
FragmentationPercent = snapshot.FragmentationPercent,
|
||||
TailReclaimablePages = snapshot.TailReclaimablePages
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets detailed page-level fragmentation diagnostics.
|
||||
/// </summary>
|
||||
public FragmentationMapReport GetFragmentationMap()
|
||||
{
|
||||
var freePageSet = new HashSet<uint>(_pageFile.EnumerateFreePages(includeEmptyPages: true));
|
||||
var pageCount = _pageFile.NextPageId;
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var pages = new List<FragmentationPageEntry>((int)pageCount);
|
||||
|
||||
long totalFreeBytes = 0;
|
||||
|
||||
for (uint pageId = 0; pageId < pageCount; pageId++)
|
||||
{
|
||||
_pageFile.ReadPage(pageId, buffer);
|
||||
var pageHeader = PageHeader.ReadFrom(buffer);
|
||||
var isFreePage = freePageSet.Contains(pageId);
|
||||
|
||||
int freeBytes = 0;
|
||||
if (isFreePage)
|
||||
{
|
||||
freeBytes = _pageFile.PageSize;
|
||||
}
|
||||
else if (TryReadSlottedFreeSpace(buffer, out var slottedFreeBytes))
|
||||
{
|
||||
freeBytes = slottedFreeBytes;
|
||||
}
|
||||
|
||||
totalFreeBytes += freeBytes;
|
||||
|
||||
pages.Add(new FragmentationPageEntry
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = pageHeader.PageType,
|
||||
IsFreePage = isFreePage,
|
||||
FreeBytes = freeBytes
|
||||
});
|
||||
}
|
||||
|
||||
uint tailReclaimablePages = 0;
|
||||
for (var i = pageCount; i > 2; i--)
|
||||
{
|
||||
if (!freePageSet.Contains(i - 1))
|
||||
break;
|
||||
|
||||
tailReclaimablePages++;
|
||||
}
|
||||
|
||||
var fileBytes = Math.Max(1L, _pageFile.FileLengthBytes);
|
||||
return new FragmentationMapReport
|
||||
{
|
||||
Pages = pages,
|
||||
TotalFreeBytes = totalFreeBytes,
|
||||
FragmentationPercent = (totalFreeBytes * 100d) / fileBytes,
|
||||
TailReclaimablePages = tailReclaimablePages
|
||||
};
|
||||
}
|
||||
|
||||
private IEnumerable<DocumentLocation> EnumeratePrimaryLocations(CollectionMetadata metadata)
|
||||
{
|
||||
if (metadata.PrimaryRootPageId == 0)
|
||||
yield break;
|
||||
|
||||
var index = new BTreeIndex(this, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
||||
|
||||
foreach (var entry in index.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transactionId: 0))
|
||||
{
|
||||
yield return entry.Location;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadFirstOverflowPage(in DocumentLocation location, out uint firstOverflowPage)
|
||||
{
|
||||
firstOverflowPage = 0;
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= header.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
if ((slot.Flags & SlotFlags.HasOverflow) == 0)
|
||||
return false;
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
firstOverflowPage = BinaryPrimitives.ReadUInt32LittleEndian(pageBuffer.AsSpan(slot.Offset + 4, 4));
|
||||
return true;
|
||||
}
|
||||
|
||||
private void AddOverflowChainPages(HashSet<uint> pageIds, uint firstOverflowPage)
|
||||
{
|
||||
if (firstOverflowPage == 0)
|
||||
return;
|
||||
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var visited = new HashSet<uint>();
|
||||
var current = firstOverflowPage;
|
||||
|
||||
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
||||
{
|
||||
pageIds.Add(current);
|
||||
_pageFile.ReadPage(current, buffer);
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
if (header.PageType != PageType.Overflow)
|
||||
break;
|
||||
|
||||
current = header.NextOverflowPage;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadSlotPayloadStats(
|
||||
in DocumentLocation location,
|
||||
out bool isCompressed,
|
||||
out int originalBytes,
|
||||
out int storedBytes)
|
||||
{
|
||||
isCompressed = false;
|
||||
originalBytes = 0;
|
||||
storedBytes = 0;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= header.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
var hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
||||
|
||||
if (!hasOverflow)
|
||||
{
|
||||
storedBytes = slot.Length;
|
||||
if (!isCompressed)
|
||||
{
|
||||
originalBytes = slot.Length;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < CompressedPayloadHeader.Size)
|
||||
return false;
|
||||
|
||||
var compressedHeader = CompressedPayloadHeader.ReadFrom(pageBuffer.AsSpan(slot.Offset, CompressedPayloadHeader.Size));
|
||||
originalBytes = compressedHeader.OriginalLength;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||
var totalStoredBytes = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
||||
if (totalStoredBytes < 0)
|
||||
return false;
|
||||
|
||||
storedBytes = totalStoredBytes;
|
||||
if (!isCompressed)
|
||||
{
|
||||
originalBytes = totalStoredBytes;
|
||||
return true;
|
||||
}
|
||||
|
||||
var storedPrefix = primaryPayload.Slice(8);
|
||||
Span<byte> headerBuffer = stackalloc byte[CompressedPayloadHeader.Size];
|
||||
|
||||
if (storedPrefix.Length >= CompressedPayloadHeader.Size)
|
||||
{
|
||||
storedPrefix.Slice(0, CompressedPayloadHeader.Size).CopyTo(headerBuffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
storedPrefix.CopyTo(headerBuffer);
|
||||
var copied = storedPrefix.Length;
|
||||
var nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
||||
var overflowBuffer = new byte[_pageFile.PageSize];
|
||||
|
||||
while (copied < CompressedPayloadHeader.Size && nextOverflow != 0 && nextOverflow < _pageFile.NextPageId)
|
||||
{
|
||||
_pageFile.ReadPage(nextOverflow, overflowBuffer);
|
||||
var overflowHeader = SlottedPageHeader.ReadFrom(overflowBuffer);
|
||||
if (overflowHeader.PageType != PageType.Overflow)
|
||||
return false;
|
||||
|
||||
var available = Math.Min(CompressedPayloadHeader.Size - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
||||
overflowBuffer.AsSpan(SlottedPageHeader.Size, available).CopyTo(headerBuffer.Slice(copied));
|
||||
copied += available;
|
||||
nextOverflow = overflowHeader.NextOverflowPage;
|
||||
}
|
||||
|
||||
if (copied < CompressedPayloadHeader.Size)
|
||||
return false;
|
||||
}
|
||||
|
||||
var headerFromPayload = CompressedPayloadHeader.ReadFrom(headerBuffer);
|
||||
originalBytes = headerFromPayload.OriginalLength;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user