Initialize CBDD solution and add a .NET-focused gitignore for generated artifacts.
This commit is contained in:
14
src/CBDD.Bson/Attributes.cs
Executable file
14
src/CBDD.Bson/Attributes.cs
Executable file
@@ -0,0 +1,14 @@
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson
|
||||
{
|
||||
[AttributeUsage(AttributeTargets.Property)]
|
||||
public class BsonIdAttribute : Attribute
|
||||
{
|
||||
}
|
||||
|
||||
[AttributeUsage(AttributeTargets.Property)]
|
||||
public class BsonIgnoreAttribute : Attribute
|
||||
{
|
||||
}
|
||||
}
|
||||
262
src/CBDD.Bson/BsonBufferWriter.cs
Executable file
262
src/CBDD.Bson/BsonBufferWriter.cs
Executable file
@@ -0,0 +1,262 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// BSON writer that serializes to an IBufferWriter, enabling streaming serialization
|
||||
/// without fixed buffer size limits.
|
||||
/// </summary>
|
||||
public ref struct BsonBufferWriter
|
||||
{
|
||||
private IBufferWriter<byte> _writer;
|
||||
private int _totalBytesWritten;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonBufferWriter"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="writer">The buffer writer to write BSON bytes to.</param>
|
||||
public BsonBufferWriter(IBufferWriter<byte> writer)
|
||||
{
|
||||
_writer = writer;
|
||||
_totalBytesWritten = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current write position in bytes.
|
||||
/// </summary>
|
||||
public int Position => _totalBytesWritten;
|
||||
|
||||
private void WriteBytes(ReadOnlySpan<byte> data)
|
||||
{
|
||||
var destination = _writer.GetSpan(data.Length);
|
||||
data.CopyTo(destination);
|
||||
_writer.Advance(data.Length);
|
||||
_totalBytesWritten += data.Length;
|
||||
}
|
||||
|
||||
private void WriteByte(byte value)
|
||||
{
|
||||
var span = _writer.GetSpan(1);
|
||||
span[0] = value;
|
||||
_writer.Advance(1);
|
||||
_totalBytesWritten++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON date-time field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The date-time value.</param>
|
||||
public void WriteDateTime(string name, DateTime value)
|
||||
{
|
||||
WriteByte((byte)BsonType.DateTime);
|
||||
WriteCString(name);
|
||||
// BSON DateTime: milliseconds since Unix epoch (UTC)
|
||||
var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var milliseconds = (long)(value.ToUniversalTime() - unixEpoch).TotalMilliseconds;
|
||||
WriteInt64Internal(milliseconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing a BSON document.
|
||||
/// </summary>
|
||||
/// <returns>The position where the document size placeholder was written.</returns>
|
||||
public int BeginDocument()
|
||||
{
|
||||
// Write placeholder for size (4 bytes)
|
||||
var sizePosition = _totalBytesWritten;
|
||||
var span = _writer.GetSpan(4);
|
||||
// Initialize with default value (will be patched later)
|
||||
span[0] = 0; span[1] = 0; span[2] = 0; span[3] = 0;
|
||||
_writer.Advance(4);
|
||||
_totalBytesWritten += 4;
|
||||
return sizePosition;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends the current BSON document by writing the document terminator.
|
||||
/// </summary>
|
||||
/// <param name="sizePosition">The position of the size placeholder for this document.</param>
|
||||
public void EndDocument(int sizePosition)
|
||||
{
|
||||
// Write document terminator
|
||||
WriteByte(0);
|
||||
|
||||
// Note: Size patching must be done by caller after accessing WrittenSpan
|
||||
// from ArrayBufferWriter (or equivalent)
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing a nested BSON document field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <returns>The position where the nested document size placeholder was written.</returns>
|
||||
public int BeginDocument(string name)
|
||||
{
|
||||
WriteByte((byte)BsonType.Document);
|
||||
WriteCString(name);
|
||||
return BeginDocument();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing a BSON array field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <returns>The position where the array document size placeholder was written.</returns>
|
||||
public int BeginArray(string name)
|
||||
{
|
||||
WriteByte((byte)BsonType.Array);
|
||||
WriteCString(name);
|
||||
return BeginDocument();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends the current BSON array.
|
||||
/// </summary>
|
||||
/// <param name="sizePosition">The position of the size placeholder for this array.</param>
|
||||
public void EndArray(int sizePosition)
|
||||
{
|
||||
EndDocument(sizePosition);
|
||||
}
|
||||
|
||||
// Private helper methods
|
||||
|
||||
private void WriteInt32Internal(int value)
|
||||
{
|
||||
var span = _writer.GetSpan(4);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(span, value);
|
||||
_writer.Advance(4);
|
||||
_totalBytesWritten += 4;
|
||||
}
|
||||
|
||||
private void WriteInt64Internal(long value)
|
||||
{
|
||||
var span = _writer.GetSpan(8);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(span, value);
|
||||
_writer.Advance(8);
|
||||
_totalBytesWritten += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON ObjectId field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The ObjectId value.</param>
|
||||
public void WriteObjectId(string name, ObjectId value)
|
||||
{
|
||||
WriteByte((byte)BsonType.ObjectId);
|
||||
WriteCString(name);
|
||||
WriteBytes(value.ToByteArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON string field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The string value.</param>
|
||||
public void WriteString(string name, string value)
|
||||
{
|
||||
WriteByte((byte)BsonType.String);
|
||||
WriteCString(name);
|
||||
WriteStringValue(value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON boolean field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The boolean value.</param>
|
||||
public void WriteBoolean(string name, bool value)
|
||||
{
|
||||
WriteByte((byte)BsonType.Boolean);
|
||||
WriteCString(name);
|
||||
WriteByte((byte)(value ? 1 : 0));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON null field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
public void WriteNull(string name)
|
||||
{
|
||||
WriteByte((byte)BsonType.Null);
|
||||
WriteCString(name);
|
||||
}
|
||||
|
||||
private void WriteStringValue(string value)
|
||||
{
|
||||
// String: length (int32) + UTF8 bytes + null terminator
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
WriteInt32Internal(bytes.Length + 1); // +1 for null terminator
|
||||
WriteBytes(bytes);
|
||||
WriteByte(0);
|
||||
}
|
||||
|
||||
private void WriteDoubleInternal(double value)
|
||||
{
|
||||
var span = _writer.GetSpan(8);
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(span, value);
|
||||
_writer.Advance(8);
|
||||
_totalBytesWritten += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON binary field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="data">The binary data.</param>
|
||||
public void WriteBinary(string name, ReadOnlySpan<byte> data)
|
||||
{
|
||||
WriteByte((byte)BsonType.Binary);
|
||||
WriteCString(name);
|
||||
WriteInt32Internal(data.Length);
|
||||
WriteByte(0); // Binary subtype: Generic
|
||||
WriteBytes(data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON 64-bit integer field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The 64-bit integer value.</param>
|
||||
public void WriteInt64(string name, long value)
|
||||
{
|
||||
WriteByte((byte)BsonType.Int64);
|
||||
WriteCString(name);
|
||||
WriteInt64Internal(value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON double field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The double value.</param>
|
||||
public void WriteDouble(string name, double value)
|
||||
{
|
||||
WriteByte((byte)BsonType.Double);
|
||||
WriteCString(name);
|
||||
WriteDoubleInternal(value);
|
||||
}
|
||||
|
||||
private void WriteCString(string value)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(value);
|
||||
WriteBytes(bytes);
|
||||
WriteByte(0); // Null terminator
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON 32-bit integer field.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The 32-bit integer value.</param>
|
||||
public void WriteInt32(string name, int value)
|
||||
{
|
||||
WriteByte((byte)BsonType.Int32);
|
||||
WriteCString(name);
|
||||
WriteInt32Internal(value);
|
||||
}
|
||||
}
|
||||
292
src/CBDD.Bson/BsonDocument.cs
Executable file
292
src/CBDD.Bson/BsonDocument.cs
Executable file
@@ -0,0 +1,292 @@
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// Represents an in-memory BSON document with lazy parsing.
|
||||
/// Uses Memory<byte> to store raw BSON data for zero-copy operations.
|
||||
/// </summary>
|
||||
public sealed class BsonDocument
|
||||
{
|
||||
private readonly Memory<byte> _rawData;
|
||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? _keys;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonDocument"/> class from raw BSON memory.
|
||||
/// </summary>
|
||||
/// <param name="rawBsonData">The raw BSON data.</param>
|
||||
/// <param name="keys">The optional key dictionary.</param>
|
||||
public BsonDocument(Memory<byte> rawBsonData, System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? keys = null)
|
||||
{
|
||||
_rawData = rawBsonData;
|
||||
_keys = keys;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonDocument"/> class from raw BSON bytes.
|
||||
/// </summary>
|
||||
/// <param name="rawBsonData">The raw BSON data.</param>
|
||||
/// <param name="keys">The optional key dictionary.</param>
|
||||
public BsonDocument(byte[] rawBsonData, System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? keys = null)
|
||||
{
|
||||
_rawData = rawBsonData;
|
||||
_keys = keys;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the raw BSON bytes
|
||||
/// </summary>
|
||||
public ReadOnlySpan<byte> RawData => _rawData.Span;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the document size in bytes
|
||||
/// </summary>
|
||||
public int Size => BitConverter.ToInt32(_rawData.Span[..4]);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a reader for this document
|
||||
/// </summary>
|
||||
public BsonSpanReader GetReader() => new BsonSpanReader(_rawData.Span, _keys ?? new System.Collections.Concurrent.ConcurrentDictionary<ushort, string>());
|
||||
|
||||
/// <summary>
|
||||
/// Tries to get a field value by name.
|
||||
/// Returns false if field not found.
|
||||
/// </summary>
|
||||
/// <param name="fieldName">The field name.</param>
|
||||
/// <param name="value">When this method returns, contains the field value if found; otherwise <see langword="null"/>.</param>
|
||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
||||
public bool TryGetString(string fieldName, out string? value)
|
||||
{
|
||||
value = null;
|
||||
var reader = GetReader();
|
||||
if (reader.Remaining < 5)
|
||||
return false;
|
||||
|
||||
_ = reader.ReadDocumentSize();
|
||||
fieldName = fieldName.ToLowerInvariant();
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var type = reader.ReadBsonType();
|
||||
if (type == BsonType.EndOfDocument)
|
||||
break;
|
||||
|
||||
var name = reader.ReadElementHeader();
|
||||
|
||||
if (name == fieldName && type == BsonType.String)
|
||||
{
|
||||
value = reader.ReadString();
|
||||
return true;
|
||||
}
|
||||
|
||||
reader.SkipValue(type);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to get an Int32 field value by name.
|
||||
/// </summary>
|
||||
/// <param name="fieldName">The field name.</param>
|
||||
/// <param name="value">When this method returns, contains the field value if found; otherwise zero.</param>
|
||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
||||
public bool TryGetInt32(string fieldName, out int value)
|
||||
{
|
||||
value = 0;
|
||||
var reader = GetReader();
|
||||
if (reader.Remaining < 5)
|
||||
return false;
|
||||
|
||||
_ = reader.ReadDocumentSize();
|
||||
fieldName = fieldName.ToLowerInvariant();
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var type = reader.ReadBsonType();
|
||||
if (type == BsonType.EndOfDocument)
|
||||
break;
|
||||
|
||||
var name = reader.ReadElementHeader();
|
||||
|
||||
if (name == fieldName && type == BsonType.Int32)
|
||||
{
|
||||
value = reader.ReadInt32();
|
||||
return true;
|
||||
}
|
||||
|
||||
reader.SkipValue(type);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to get an ObjectId field value by name.
|
||||
/// </summary>
|
||||
/// <param name="fieldName">The field name.</param>
|
||||
/// <param name="value">When this method returns, contains the field value if found; otherwise default.</param>
|
||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
||||
public bool TryGetObjectId(string fieldName, out ObjectId value)
|
||||
{
|
||||
value = default;
|
||||
var reader = GetReader();
|
||||
if (reader.Remaining < 5)
|
||||
return false;
|
||||
|
||||
_ = reader.ReadDocumentSize();
|
||||
fieldName = fieldName.ToLowerInvariant();
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var type = reader.ReadBsonType();
|
||||
if (type == BsonType.EndOfDocument)
|
||||
break;
|
||||
|
||||
var name = reader.ReadElementHeader();
|
||||
|
||||
if (name == fieldName && type == BsonType.ObjectId)
|
||||
{
|
||||
value = reader.ReadObjectId();
|
||||
return true;
|
||||
}
|
||||
|
||||
reader.SkipValue(type);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new BsonDocument from field values using a builder pattern
|
||||
/// </summary>
|
||||
/// <param name="keyMap">The key map used for field name encoding.</param>
|
||||
/// <param name="buildAction">The action that populates the builder.</param>
|
||||
/// <returns>The created BSON document.</returns>
|
||||
public static BsonDocument Create(System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap, Action<BsonDocumentBuilder> buildAction)
|
||||
{
|
||||
var builder = new BsonDocumentBuilder(keyMap);
|
||||
buildAction(builder);
|
||||
return builder.Build();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builder for creating BSON documents
|
||||
/// </summary>
|
||||
public sealed class BsonDocumentBuilder
|
||||
{
|
||||
private byte[] _buffer = new byte[1024]; // Start with 1KB
|
||||
private int _position;
|
||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, ushort> _keyMap;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonDocumentBuilder"/> class.
|
||||
/// </summary>
|
||||
/// <param name="keyMap">The key map used for field name encoding.</param>
|
||||
public BsonDocumentBuilder(System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap)
|
||||
{
|
||||
_keyMap = keyMap;
|
||||
var writer = new BsonSpanWriter(_buffer, _keyMap);
|
||||
_position = writer.Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a string field to the document.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The field value.</param>
|
||||
/// <returns>The current builder.</returns>
|
||||
public BsonDocumentBuilder AddString(string name, string value)
|
||||
{
|
||||
EnsureCapacity(256); // Conservative estimate
|
||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||
writer.WriteString(name, value);
|
||||
_position += writer.Position;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an Int32 field to the document.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The field value.</param>
|
||||
/// <returns>The current builder.</returns>
|
||||
public BsonDocumentBuilder AddInt32(string name, int value)
|
||||
{
|
||||
EnsureCapacity(64);
|
||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||
writer.WriteInt32(name, value);
|
||||
_position += writer.Position;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an Int64 field to the document.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The field value.</param>
|
||||
/// <returns>The current builder.</returns>
|
||||
public BsonDocumentBuilder AddInt64(string name, long value)
|
||||
{
|
||||
EnsureCapacity(64);
|
||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||
writer.WriteInt64(name, value);
|
||||
_position += writer.Position;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a Boolean field to the document.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The field value.</param>
|
||||
/// <returns>The current builder.</returns>
|
||||
public BsonDocumentBuilder AddBoolean(string name, bool value)
|
||||
{
|
||||
EnsureCapacity(64);
|
||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||
writer.WriteBoolean(name, value);
|
||||
_position += writer.Position;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an ObjectId field to the document.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The field value.</param>
|
||||
/// <returns>The current builder.</returns>
|
||||
public BsonDocumentBuilder AddObjectId(string name, ObjectId value)
|
||||
{
|
||||
EnsureCapacity(64);
|
||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||
writer.WriteObjectId(name, value);
|
||||
_position += writer.Position;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a BSON document from the accumulated fields.
|
||||
/// </summary>
|
||||
/// <returns>The constructed BSON document.</returns>
|
||||
public BsonDocument Build()
|
||||
{
|
||||
// Layout: [int32 size][field bytes...][0x00 terminator]
|
||||
var totalSize = _position + 5;
|
||||
var finalBuffer = new byte[totalSize];
|
||||
|
||||
BitConverter.TryWriteBytes(finalBuffer.AsSpan(0, 4), totalSize);
|
||||
_buffer.AsSpan(0, _position).CopyTo(finalBuffer.AsSpan(4, _position));
|
||||
finalBuffer[totalSize - 1] = 0;
|
||||
|
||||
return new BsonDocument(finalBuffer);
|
||||
}
|
||||
|
||||
private void EnsureCapacity(int additional)
|
||||
{
|
||||
if (_position + additional > _buffer.Length)
|
||||
{
|
||||
var newBuffer = new byte[_buffer.Length * 2];
|
||||
_buffer.CopyTo(newBuffer, 0);
|
||||
_buffer = newBuffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
396
src/CBDD.Bson/BsonSpanReader.cs
Executable file
396
src/CBDD.Bson/BsonSpanReader.cs
Executable file
@@ -0,0 +1,396 @@
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// Zero-allocation BSON reader using ReadOnlySpan<byte>.
|
||||
/// Implemented as ref struct to ensure stack-only allocation.
|
||||
/// </summary>
|
||||
public ref struct BsonSpanReader
|
||||
{
|
||||
private ReadOnlySpan<byte> _buffer;
|
||||
private int _position;
|
||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<ushort, string> _keys;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonSpanReader"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="buffer">The BSON buffer to read.</param>
|
||||
/// <param name="keys">The reverse key dictionary used for compressed element headers.</param>
|
||||
public BsonSpanReader(ReadOnlySpan<byte> buffer, System.Collections.Concurrent.ConcurrentDictionary<ushort, string> keys)
|
||||
{
|
||||
_buffer = buffer;
|
||||
_position = 0;
|
||||
_keys = keys;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current read position in the buffer.
|
||||
/// </summary>
|
||||
public int Position => _position;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of unread bytes remaining in the buffer.
|
||||
/// </summary>
|
||||
public int Remaining => _buffer.Length - _position;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the document size (first 4 bytes of a BSON document)
|
||||
/// </summary>
|
||||
public int ReadDocumentSize()
|
||||
{
|
||||
if (Remaining < 4)
|
||||
throw new InvalidOperationException("Not enough bytes to read document size");
|
||||
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
_position += 4;
|
||||
return size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a BSON element type
|
||||
/// </summary>
|
||||
public BsonType ReadBsonType()
|
||||
{
|
||||
if (Remaining < 1)
|
||||
throw new InvalidOperationException("Not enough bytes to read BSON type");
|
||||
|
||||
var type = (BsonType)_buffer[_position];
|
||||
_position++;
|
||||
return type;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a C-style null-terminated string (e-name in BSON spec)
|
||||
/// </summary>
|
||||
public string ReadCString()
|
||||
{
|
||||
var start = _position;
|
||||
while (_position < _buffer.Length && _buffer[_position] != 0)
|
||||
_position++;
|
||||
|
||||
if (_position >= _buffer.Length)
|
||||
throw new InvalidOperationException("Unterminated C-string");
|
||||
|
||||
var nameBytes = _buffer.Slice(start, _position - start);
|
||||
_position++; // Skip null terminator
|
||||
|
||||
return Encoding.UTF8.GetString(nameBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a C-string into a destination span. Returns the number of bytes written.
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination character span.</param>
|
||||
public int ReadCString(Span<char> destination)
|
||||
{
|
||||
var start = _position;
|
||||
while (_position < _buffer.Length && _buffer[_position] != 0)
|
||||
_position++;
|
||||
|
||||
if (_position >= _buffer.Length)
|
||||
throw new InvalidOperationException("Unterminated C-string");
|
||||
|
||||
var nameBytes = _buffer.Slice(start, _position - start);
|
||||
_position++; // Skip null terminator
|
||||
|
||||
return Encoding.UTF8.GetChars(nameBytes, destination);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a BSON string (4-byte length + UTF-8 bytes + null terminator)
|
||||
/// </summary>
|
||||
public string ReadString()
|
||||
{
|
||||
var length = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
_position += 4;
|
||||
|
||||
if (length < 1)
|
||||
throw new InvalidOperationException("Invalid string length");
|
||||
|
||||
var stringBytes = _buffer.Slice(_position, length - 1); // Exclude null terminator
|
||||
_position += length;
|
||||
|
||||
return Encoding.UTF8.GetString(stringBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a 32-bit integer.
|
||||
/// </summary>
|
||||
public int ReadInt32()
|
||||
{
|
||||
if (Remaining < 4)
|
||||
throw new InvalidOperationException("Not enough bytes to read Int32");
|
||||
|
||||
var value = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
_position += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a 64-bit integer.
|
||||
/// </summary>
|
||||
public long ReadInt64()
|
||||
{
|
||||
if (Remaining < 8)
|
||||
throw new InvalidOperationException("Not enough bytes to read Int64");
|
||||
|
||||
var value = BinaryPrimitives.ReadInt64LittleEndian(_buffer.Slice(_position, 8));
|
||||
_position += 8;
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a double-precision floating point value.
|
||||
/// </summary>
|
||||
public double ReadDouble()
|
||||
{
|
||||
if (Remaining < 8)
|
||||
throw new InvalidOperationException("Not enough bytes to read Double");
|
||||
|
||||
var value = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
||||
_position += 8;
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads spatial coordinates from a BSON array [X, Y].
|
||||
/// Returns a (double, double) tuple.
|
||||
/// </summary>
|
||||
public (double, double) ReadCoordinates()
|
||||
{
|
||||
// Skip array size (4 bytes)
|
||||
_position += 4;
|
||||
|
||||
// Skip element 0 header: Type(1) + Name("0\0") (3 bytes)
|
||||
_position += 3;
|
||||
var x = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
||||
_position += 8;
|
||||
|
||||
// Skip element 1 header: Type(1) + Name("1\0") (3 bytes)
|
||||
_position += 3;
|
||||
var y = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
||||
_position += 8;
|
||||
|
||||
// Skip end of array marker (1 byte)
|
||||
_position++;
|
||||
|
||||
return (x, y);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a Decimal128 value.
|
||||
/// </summary>
|
||||
public decimal ReadDecimal128()
|
||||
{
|
||||
if (Remaining < 16)
|
||||
throw new InvalidOperationException("Not enough bytes to read Decimal128");
|
||||
|
||||
var bits = new int[4];
|
||||
bits[0] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
bits[1] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 4, 4));
|
||||
bits[2] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 8, 4));
|
||||
bits[3] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 12, 4));
|
||||
_position += 16;
|
||||
|
||||
return new decimal(bits);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a boolean value.
|
||||
/// </summary>
|
||||
public bool ReadBoolean()
|
||||
{
|
||||
if (Remaining < 1)
|
||||
throw new InvalidOperationException("Not enough bytes to read Boolean");
|
||||
|
||||
var value = _buffer[_position] != 0;
|
||||
_position++;
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a BSON DateTime (UTC milliseconds since Unix epoch)
|
||||
/// </summary>
|
||||
public DateTime ReadDateTime()
|
||||
{
|
||||
var milliseconds = ReadInt64();
|
||||
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds).UtcDateTime;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a BSON DateTime as DateTimeOffset (UTC milliseconds since Unix epoch)
|
||||
/// </summary>
|
||||
public DateTimeOffset ReadDateTimeOffset()
|
||||
{
|
||||
var milliseconds = ReadInt64();
|
||||
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a TimeSpan from BSON Int64 (ticks)
|
||||
/// </summary>
|
||||
public TimeSpan ReadTimeSpan()
|
||||
{
|
||||
var ticks = ReadInt64();
|
||||
return TimeSpan.FromTicks(ticks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a DateOnly from BSON Int32 (day number)
|
||||
/// </summary>
|
||||
public DateOnly ReadDateOnly()
|
||||
{
|
||||
var dayNumber = ReadInt32();
|
||||
return DateOnly.FromDayNumber(dayNumber);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a TimeOnly from BSON Int64 (ticks)
|
||||
/// </summary>
|
||||
public TimeOnly ReadTimeOnly()
|
||||
{
|
||||
var ticks = ReadInt64();
|
||||
return new TimeOnly(ticks);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a GUID value.
|
||||
/// </summary>
|
||||
public Guid ReadGuid()
|
||||
{
|
||||
return Guid.Parse(ReadString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a BSON ObjectId (12 bytes)
|
||||
/// </summary>
|
||||
public ObjectId ReadObjectId()
|
||||
{
|
||||
if (Remaining < 12)
|
||||
throw new InvalidOperationException("Not enough bytes to read ObjectId");
|
||||
|
||||
var oidBytes = _buffer.Slice(_position, 12);
|
||||
_position += 12;
|
||||
return new ObjectId(oidBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads binary data (subtype + length + bytes)
|
||||
/// </summary>
|
||||
/// <param name="subtype">When this method returns, contains the BSON binary subtype.</param>
|
||||
public ReadOnlySpan<byte> ReadBinary(out byte subtype)
|
||||
{
|
||||
var length = ReadInt32();
|
||||
|
||||
if (Remaining < 1)
|
||||
throw new InvalidOperationException("Not enough bytes to read binary subtype");
|
||||
|
||||
subtype = _buffer[_position];
|
||||
_position++;
|
||||
|
||||
if (Remaining < length)
|
||||
throw new InvalidOperationException("Not enough bytes to read binary data");
|
||||
|
||||
var data = _buffer.Slice(_position, length);
|
||||
_position += length;
|
||||
return data;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Skips the current value based on type
|
||||
/// </summary>
|
||||
/// <param name="type">The BSON type of the value to skip.</param>
|
||||
public void SkipValue(BsonType type)
|
||||
{
|
||||
switch (type)
|
||||
{
|
||||
case BsonType.Double:
|
||||
_position += 8;
|
||||
break;
|
||||
case BsonType.String:
|
||||
var stringLength = ReadInt32();
|
||||
_position += stringLength;
|
||||
break;
|
||||
case BsonType.Document:
|
||||
case BsonType.Array:
|
||||
var docLength = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
_position += docLength;
|
||||
break;
|
||||
case BsonType.Binary:
|
||||
var binaryLength = ReadInt32();
|
||||
_position += 1 + binaryLength; // subtype + data
|
||||
break;
|
||||
case BsonType.ObjectId:
|
||||
_position += 12;
|
||||
break;
|
||||
case BsonType.Boolean:
|
||||
_position += 1;
|
||||
break;
|
||||
case BsonType.DateTime:
|
||||
case BsonType.Int64:
|
||||
case BsonType.Timestamp:
|
||||
_position += 8;
|
||||
break;
|
||||
case BsonType.Decimal128:
|
||||
_position += 16;
|
||||
break;
|
||||
case BsonType.Int32:
|
||||
_position += 4;
|
||||
break;
|
||||
case BsonType.Null:
|
||||
// No data
|
||||
break;
|
||||
default:
|
||||
throw new NotSupportedException($"Skipping type {type} not supported");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a single byte.
|
||||
/// </summary>
|
||||
public byte ReadByte()
|
||||
{
|
||||
if (Remaining < 1)
|
||||
throw new InvalidOperationException("Not enough bytes to read byte");
|
||||
var value = _buffer[_position];
|
||||
_position++;
|
||||
return value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Peeks a 32-bit integer at the current position without advancing.
|
||||
/// </summary>
|
||||
public int PeekInt32()
|
||||
{
|
||||
if (Remaining < 4)
|
||||
throw new InvalidOperationException("Not enough bytes to peek Int32");
|
||||
return BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads an element header key identifier and resolves it to a key name.
|
||||
/// </summary>
|
||||
public string ReadElementHeader()
|
||||
{
|
||||
if (Remaining < 2)
|
||||
throw new InvalidOperationException("Not enough bytes to read BSON element key ID");
|
||||
|
||||
var id = BinaryPrimitives.ReadUInt16LittleEndian(_buffer.Slice(_position, 2));
|
||||
_position += 2;
|
||||
|
||||
if (!_keys.TryGetValue(id, out var key))
|
||||
{
|
||||
throw new InvalidOperationException($"BSON Key ID {id} not found in reverse key dictionary.");
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a span containing all unread bytes.
|
||||
/// </summary>
|
||||
public ReadOnlySpan<byte> RemainingBytes() => _buffer[_position..];
|
||||
}
|
||||
382
src/CBDD.Bson/BsonSpanWriter.cs
Executable file
382
src/CBDD.Bson/BsonSpanWriter.cs
Executable file
@@ -0,0 +1,382 @@
|
||||
using System;
|
||||
using System.Buffers.Binary;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// Zero-allocation BSON writer using Span<byte>.
|
||||
/// Implemented as ref struct to ensure stack-only allocation.
|
||||
/// </summary>
|
||||
public ref struct BsonSpanWriter
|
||||
{
|
||||
private Span<byte> _buffer;
|
||||
private int _position;
|
||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, ushort> _keyMap;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BsonSpanWriter"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="buffer">The destination buffer to write BSON bytes into.</param>
|
||||
/// <param name="keyMap">The cached key-name to key-id mapping.</param>
|
||||
public BsonSpanWriter(Span<byte> buffer, System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap)
|
||||
{
|
||||
_buffer = buffer;
|
||||
_keyMap = keyMap;
|
||||
_position = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current write position in the buffer.
|
||||
/// </summary>
|
||||
public int Position => _position;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of bytes remaining in the buffer.
|
||||
/// </summary>
|
||||
public int Remaining => _buffer.Length - _position;
|
||||
|
||||
/// <summary>
|
||||
/// Writes document size placeholder and returns the position to patch later
|
||||
/// </summary>
|
||||
public int WriteDocumentSizePlaceholder()
|
||||
{
|
||||
var sizePosition = _position;
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), 0);
|
||||
_position += 4;
|
||||
return sizePosition;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Patches the document size at the given position
|
||||
/// </summary>
|
||||
/// <param name="sizePosition">The position where the size placeholder was written.</param>
|
||||
public void PatchDocumentSize(int sizePosition)
|
||||
{
|
||||
var size = _position - sizePosition;
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(sizePosition, 4), size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON element header (type + name)
|
||||
/// </summary>
|
||||
/// <param name="type">The BSON element type.</param>
|
||||
/// <param name="name">The field name.</param>
|
||||
public void WriteElementHeader(BsonType type, string name)
|
||||
{
|
||||
_buffer[_position] = (byte)type;
|
||||
_position++;
|
||||
|
||||
if (!_keyMap.TryGetValue(name, out var id))
|
||||
{
|
||||
throw new InvalidOperationException($"BSON Key '{name}' not found in dictionary cache. Ensure all keys are registered before serialization.");
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(_buffer.Slice(_position, 2), id);
|
||||
_position += 2;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a C-style null-terminated string
|
||||
/// </summary>
|
||||
private void WriteCString(string value)
|
||||
{
|
||||
var bytesWritten = Encoding.UTF8.GetBytes(value, _buffer[_position..]);
|
||||
_position += bytesWritten;
|
||||
_buffer[_position] = 0; // Null terminator
|
||||
_position++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes end-of-document marker
|
||||
/// </summary>
|
||||
public void WriteEndOfDocument()
|
||||
{
|
||||
_buffer[_position] = 0;
|
||||
_position++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON string element
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The string value.</param>
|
||||
public void WriteString(string name, string value)
|
||||
{
|
||||
WriteElementHeader(BsonType.String, name);
|
||||
|
||||
var valueBytes = Encoding.UTF8.GetByteCount(value);
|
||||
var stringLength = valueBytes + 1; // Include null terminator
|
||||
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), stringLength);
|
||||
_position += 4;
|
||||
|
||||
Encoding.UTF8.GetBytes(value, _buffer[_position..]);
|
||||
_position += valueBytes;
|
||||
|
||||
_buffer[_position] = 0; // Null terminator
|
||||
_position++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON int32 element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The 32-bit integer value.</param>
|
||||
public void WriteInt32(string name, int value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Int32, name);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), value);
|
||||
_position += 4;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON int64 element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The 64-bit integer value.</param>
|
||||
public void WriteInt64(string name, long value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Int64, name);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON double element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The double-precision value.</param>
|
||||
public void WriteDouble(string name, double value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Double, name);
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), value);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes spatial coordinates as a BSON array [X, Y].
|
||||
/// Optimized for (double, double) tuples.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="coordinates">The coordinate tuple as (X, Y).</param>
|
||||
public void WriteCoordinates(string name, (double, double) coordinates)
|
||||
{
|
||||
WriteElementHeader(BsonType.Array, name);
|
||||
|
||||
var startPos = _position;
|
||||
_position += 4; // Placeholder for array size
|
||||
|
||||
// Element 0: X
|
||||
_buffer[_position++] = (byte)BsonType.Double;
|
||||
_buffer[_position++] = 0x30; // '0'
|
||||
_buffer[_position++] = 0x00; // Null
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), coordinates.Item1);
|
||||
_position += 8;
|
||||
|
||||
// Element 1: Y
|
||||
_buffer[_position++] = (byte)BsonType.Double;
|
||||
_buffer[_position++] = 0x31; // '1'
|
||||
_buffer[_position++] = 0x00; // Null
|
||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), coordinates.Item2);
|
||||
_position += 8;
|
||||
|
||||
_buffer[_position++] = 0x00; // End of array marker
|
||||
|
||||
// Patch array size
|
||||
var size = _position - startPos;
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(startPos, 4), size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON Decimal128 element from a <see cref="decimal"/> value.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The decimal value.</param>
|
||||
public void WriteDecimal128(string name, decimal value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Decimal128, name);
|
||||
// Note: usage of C# decimal bits for round-trip fidelity within ZB.MOM.WW.CBDD.
|
||||
// This makes it compatible with CBDD Reader but strictly speaking not standard IEEE 754-2008 Decimal128.
|
||||
var bits = decimal.GetBits(value);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), bits[0]);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 4, 4), bits[1]);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 8, 4), bits[2]);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 12, 4), bits[3]);
|
||||
_position += 16;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON boolean element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The boolean value.</param>
|
||||
public void WriteBoolean(string name, bool value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Boolean, name);
|
||||
_buffer[_position] = (byte)(value ? 1 : 0);
|
||||
_position++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON UTC datetime element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The date and time value.</param>
|
||||
public void WriteDateTime(string name, DateTime value)
|
||||
{
|
||||
WriteElementHeader(BsonType.DateTime, name);
|
||||
var milliseconds = new DateTimeOffset(value.ToUniversalTime()).ToUnixTimeMilliseconds();
|
||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON UTC datetime element from a <see cref="DateTimeOffset"/> value.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The date and time offset value.</param>
|
||||
public void WriteDateTimeOffset(string name, DateTimeOffset value)
|
||||
{
|
||||
WriteElementHeader(BsonType.DateTime, name);
|
||||
var milliseconds = value.ToUnixTimeMilliseconds();
|
||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON int64 element containing ticks from a <see cref="TimeSpan"/>.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The time span value.</param>
|
||||
public void WriteTimeSpan(string name, TimeSpan value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Int64, name);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value.Ticks);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON int32 element containing the <see cref="DateOnly.DayNumber"/>.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The date-only value.</param>
|
||||
public void WriteDateOnly(string name, DateOnly value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Int32, name);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), value.DayNumber);
|
||||
_position += 4;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON int64 element containing ticks from a <see cref="TimeOnly"/>.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The time-only value.</param>
|
||||
public void WriteTimeOnly(string name, TimeOnly value)
|
||||
{
|
||||
WriteElementHeader(BsonType.Int64, name);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value.Ticks);
|
||||
_position += 8;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a GUID as a BSON string element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The GUID value.</param>
|
||||
public void WriteGuid(string name, Guid value)
|
||||
{
|
||||
WriteString(name, value.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON ObjectId element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="value">The ObjectId value.</param>
|
||||
public void WriteObjectId(string name, ObjectId value)
|
||||
{
|
||||
WriteElementHeader(BsonType.ObjectId, name);
|
||||
value.WriteTo(_buffer.Slice(_position, 12));
|
||||
_position += 12;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a BSON null element.
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
public void WriteNull(string name)
|
||||
{
|
||||
WriteElementHeader(BsonType.Null, name);
|
||||
// No value to write for null
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes binary data
|
||||
/// </summary>
|
||||
/// <param name="name">The field name.</param>
|
||||
/// <param name="data">The binary payload.</param>
|
||||
/// <param name="subtype">The BSON binary subtype.</param>
|
||||
public void WriteBinary(string name, ReadOnlySpan<byte> data, byte subtype = 0)
|
||||
{
|
||||
WriteElementHeader(BsonType.Binary, name);
|
||||
|
||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), data.Length);
|
||||
_position += 4;
|
||||
|
||||
_buffer[_position] = subtype;
|
||||
_position++;
|
||||
|
||||
data.CopyTo(_buffer[_position..]);
|
||||
_position += data.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing a subdocument and returns the size position to patch later
|
||||
/// </summary>
|
||||
/// <param name="name">The field name for the subdocument.</param>
|
||||
public int BeginDocument(string name)
|
||||
{
|
||||
WriteElementHeader(BsonType.Document, name);
|
||||
return WriteDocumentSizePlaceholder();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing the root document and returns the size position to patch later
|
||||
/// </summary>
|
||||
public int BeginDocument()
|
||||
{
|
||||
return WriteDocumentSizePlaceholder();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends the current document
|
||||
/// </summary>
|
||||
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder"/>.</param>
|
||||
public void EndDocument(int sizePosition)
|
||||
{
|
||||
WriteEndOfDocument();
|
||||
PatchDocumentSize(sizePosition);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins writing a BSON array and returns the size position to patch later
|
||||
/// </summary>
|
||||
/// <param name="name">The field name for the array.</param>
|
||||
public int BeginArray(string name)
|
||||
{
|
||||
WriteElementHeader(BsonType.Array, name);
|
||||
return WriteDocumentSizePlaceholder();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends the current BSON array
|
||||
/// </summary>
|
||||
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder"/>.</param>
|
||||
public void EndArray(int sizePosition)
|
||||
{
|
||||
WriteEndOfDocument();
|
||||
PatchDocumentSize(sizePosition);
|
||||
}
|
||||
}
|
||||
30
src/CBDD.Bson/BsonType.cs
Executable file
30
src/CBDD.Bson/BsonType.cs
Executable file
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// BSON type codes as defined in BSON spec
|
||||
/// </summary>
|
||||
public enum BsonType : byte
|
||||
{
|
||||
EndOfDocument = 0x00,
|
||||
Double = 0x01,
|
||||
String = 0x02,
|
||||
Document = 0x03,
|
||||
Array = 0x04,
|
||||
Binary = 0x05,
|
||||
Undefined = 0x06, // Deprecated
|
||||
ObjectId = 0x07,
|
||||
Boolean = 0x08,
|
||||
DateTime = 0x09,
|
||||
Null = 0x0A,
|
||||
Regex = 0x0B,
|
||||
DBPointer = 0x0C, // Deprecated
|
||||
JavaScript = 0x0D,
|
||||
Symbol = 0x0E, // Deprecated
|
||||
JavaScriptWithScope = 0x0F,
|
||||
Int32 = 0x10,
|
||||
Timestamp = 0x11,
|
||||
Int64 = 0x12,
|
||||
Decimal128 = 0x13,
|
||||
MinKey = 0xFF,
|
||||
MaxKey = 0x7F
|
||||
}
|
||||
112
src/CBDD.Bson/ObjectId.cs
Executable file
112
src/CBDD.Bson/ObjectId.cs
Executable file
@@ -0,0 +1,112 @@
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
/// <summary>
|
||||
/// 12-byte ObjectId compatible with MongoDB ObjectId.
|
||||
/// Implemented as readonly struct for zero allocation.
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 12)]
|
||||
public readonly struct ObjectId : IEquatable<ObjectId>
|
||||
{
|
||||
[FieldOffset(0)] private readonly int _timestamp;
|
||||
[FieldOffset(4)] private readonly long _randomAndCounter;
|
||||
|
||||
/// <summary>
|
||||
/// Empty ObjectId (all zeros)
|
||||
/// </summary>
|
||||
public static readonly ObjectId Empty = new ObjectId(0, 0);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum ObjectId (all 0xFF bytes) - useful for range queries
|
||||
/// </summary>
|
||||
public static readonly ObjectId MaxValue = new ObjectId(int.MaxValue, long.MaxValue);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ObjectId"/> struct from raw bytes.
|
||||
/// </summary>
|
||||
/// <param name="bytes">The 12-byte ObjectId value.</param>
|
||||
public ObjectId(ReadOnlySpan<byte> bytes)
|
||||
{
|
||||
if (bytes.Length != 12)
|
||||
throw new ArgumentException("ObjectId must be exactly 12 bytes", nameof(bytes));
|
||||
|
||||
_timestamp = BitConverter.ToInt32(bytes[..4]);
|
||||
_randomAndCounter = BitConverter.ToInt64(bytes[4..12]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ObjectId"/> struct from its components.
|
||||
/// </summary>
|
||||
/// <param name="timestamp">The Unix timestamp portion.</param>
|
||||
/// <param name="randomAndCounter">The random and counter portion.</param>
|
||||
public ObjectId(int timestamp, long randomAndCounter)
|
||||
{
|
||||
_timestamp = timestamp;
|
||||
_randomAndCounter = randomAndCounter;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new ObjectId with current timestamp
|
||||
/// </summary>
|
||||
public static ObjectId NewObjectId()
|
||||
{
|
||||
var timestamp = (int)DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
||||
var random = Random.Shared.NextInt64();
|
||||
return new ObjectId(timestamp, random);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the ObjectId to the destination span (must be 12 bytes)
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span to write into.</param>
|
||||
public void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < 12)
|
||||
throw new ArgumentException("Destination must be at least 12 bytes", nameof(destination));
|
||||
|
||||
BitConverter.TryWriteBytes(destination[..4], _timestamp);
|
||||
BitConverter.TryWriteBytes(destination[4..12], _randomAndCounter);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts ObjectId to byte array
|
||||
/// </summary>
|
||||
public byte[] ToByteArray()
|
||||
{
|
||||
var bytes = new byte[12];
|
||||
WriteTo(bytes);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets timestamp portion as UTC DateTime
|
||||
/// </summary>
|
||||
public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(_timestamp).UtcDateTime;
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this instance and another <see cref="ObjectId"/> have the same value.
|
||||
/// </summary>
|
||||
/// <param name="other">The object to compare with this instance.</param>
|
||||
/// <returns><see langword="true"/> if the values are equal; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Equals(ObjectId other) =>
|
||||
_timestamp == other._timestamp && _randomAndCounter == other._randomAndCounter;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool Equals(object? obj) => obj is ObjectId other && Equals(other);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int GetHashCode() => HashCode.Combine(_timestamp, _randomAndCounter);
|
||||
|
||||
public static bool operator ==(ObjectId left, ObjectId right) => left.Equals(right);
|
||||
public static bool operator !=(ObjectId left, ObjectId right) => !left.Equals(right);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString()
|
||||
{
|
||||
Span<byte> bytes = stackalloc byte[12];
|
||||
WriteTo(bytes);
|
||||
return Convert.ToHexString(bytes).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
128
src/CBDD.Bson/Schema/BsonField.cs
Executable file
128
src/CBDD.Bson/Schema/BsonField.cs
Executable file
@@ -0,0 +1,128 @@
|
||||
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
public partial class BsonField
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the field name.
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the field BSON type.
|
||||
/// </summary>
|
||||
public BsonType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the field is nullable.
|
||||
/// </summary>
|
||||
public bool IsNullable { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the nested schema when this field is a document.
|
||||
/// </summary>
|
||||
public BsonSchema? NestedSchema { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the array item type when this field is an array.
|
||||
/// </summary>
|
||||
public BsonType? ArrayItemType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Writes this field definition to BSON.
|
||||
/// </summary>
|
||||
/// <param name="writer">The BSON writer.</param>
|
||||
public void ToBson(ref BsonSpanWriter writer)
|
||||
{
|
||||
var size = writer.BeginDocument();
|
||||
writer.WriteString("n", Name);
|
||||
writer.WriteInt32("t", (int)Type);
|
||||
writer.WriteBoolean("b", IsNullable);
|
||||
|
||||
if (NestedSchema != null)
|
||||
{
|
||||
writer.WriteElementHeader(BsonType.Document, "s");
|
||||
NestedSchema.ToBson(ref writer);
|
||||
}
|
||||
|
||||
if (ArrayItemType != null)
|
||||
{
|
||||
writer.WriteInt32("a", (int)ArrayItemType.Value);
|
||||
}
|
||||
|
||||
writer.EndDocument(size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a field definition from BSON.
|
||||
/// </summary>
|
||||
/// <param name="reader">The BSON reader.</param>
|
||||
/// <returns>The deserialized field.</returns>
|
||||
public static BsonField FromBson(ref BsonSpanReader reader)
|
||||
{
|
||||
reader.ReadInt32(); // Read doc size
|
||||
|
||||
string name = "";
|
||||
BsonType type = BsonType.Null;
|
||||
bool isNullable = false;
|
||||
BsonSchema? nestedSchema = null;
|
||||
BsonType? arrayItemType = null;
|
||||
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var btype = reader.ReadBsonType();
|
||||
if (btype == BsonType.EndOfDocument) break;
|
||||
|
||||
var key = reader.ReadElementHeader();
|
||||
switch (key)
|
||||
{
|
||||
case "n": name = reader.ReadString(); break;
|
||||
case "t": type = (BsonType)reader.ReadInt32(); break;
|
||||
case "b": isNullable = reader.ReadBoolean(); break;
|
||||
case "s": nestedSchema = BsonSchema.FromBson(ref reader); break;
|
||||
case "a": arrayItemType = (BsonType)reader.ReadInt32(); break;
|
||||
default: reader.SkipValue(btype); break;
|
||||
}
|
||||
}
|
||||
|
||||
return new BsonField
|
||||
{
|
||||
Name = name,
|
||||
Type = type,
|
||||
IsNullable = isNullable,
|
||||
NestedSchema = nestedSchema,
|
||||
ArrayItemType = arrayItemType
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes a hash representing the field definition.
|
||||
/// </summary>
|
||||
/// <returns>The computed hash value.</returns>
|
||||
public long GetHash()
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(Name);
|
||||
hash.Add((int)Type);
|
||||
hash.Add(IsNullable);
|
||||
hash.Add(ArrayItemType);
|
||||
if (NestedSchema != null) hash.Add(NestedSchema.GetHash());
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this field is equal to another field.
|
||||
/// </summary>
|
||||
/// <param name="other">The other field.</param>
|
||||
/// <returns><see langword="true"/> if the fields are equal; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Equals(BsonField? other)
|
||||
{
|
||||
if (other == null) return false;
|
||||
return GetHash() == other.GetHash();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool Equals(object? obj) => Equals(obj as BsonField);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int GetHashCode() => (int)GetHash();
|
||||
}
|
||||
129
src/CBDD.Bson/Schema/BsonSchema.cs
Executable file
129
src/CBDD.Bson/Schema/BsonSchema.cs
Executable file
@@ -0,0 +1,129 @@
|
||||
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
public partial class BsonSchema
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the schema title.
|
||||
/// </summary>
|
||||
public string? Title { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the schema version.
|
||||
/// </summary>
|
||||
public int? Version { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schema fields.
|
||||
/// </summary>
|
||||
public List<BsonField> Fields { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Serializes this schema instance to BSON.
|
||||
/// </summary>
|
||||
/// <param name="writer">The BSON writer to write into.</param>
|
||||
public void ToBson(ref BsonSpanWriter writer)
|
||||
{
|
||||
var size = writer.BeginDocument();
|
||||
if (Title != null) writer.WriteString("t", Title);
|
||||
if (Version != null) writer.WriteInt32("_v", Version.Value);
|
||||
|
||||
var fieldsSize = writer.BeginArray("f");
|
||||
for (int i = 0; i < Fields.Count; i++)
|
||||
{
|
||||
writer.WriteElementHeader(BsonType.Document, i.ToString());
|
||||
Fields[i].ToBson(ref writer);
|
||||
}
|
||||
writer.EndArray(fieldsSize);
|
||||
|
||||
writer.EndDocument(size);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserializes a schema instance from BSON.
|
||||
/// </summary>
|
||||
/// <param name="reader">The BSON reader to read from.</param>
|
||||
/// <returns>The deserialized schema.</returns>
|
||||
public static BsonSchema FromBson(ref BsonSpanReader reader)
|
||||
{
|
||||
reader.ReadInt32(); // Read doc size
|
||||
|
||||
var schema = new BsonSchema();
|
||||
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var btype = reader.ReadBsonType();
|
||||
if (btype == BsonType.EndOfDocument) break;
|
||||
|
||||
var key = reader.ReadElementHeader();
|
||||
switch (key)
|
||||
{
|
||||
case "t": schema.Title = reader.ReadString(); break;
|
||||
case "_v": schema.Version = reader.ReadInt32(); break;
|
||||
case "f":
|
||||
reader.ReadInt32(); // array size
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var itemType = reader.ReadBsonType();
|
||||
if (itemType == BsonType.EndOfDocument) break;
|
||||
reader.ReadElementHeader(); // index
|
||||
schema.Fields.Add(BsonField.FromBson(ref reader));
|
||||
}
|
||||
break;
|
||||
default: reader.SkipValue(btype); break;
|
||||
}
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes a hash value for this schema based on its contents.
|
||||
/// </summary>
|
||||
/// <returns>The computed hash value.</returns>
|
||||
public long GetHash()
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.Add(Title);
|
||||
foreach (var field in Fields)
|
||||
{
|
||||
hash.Add(field.GetHash());
|
||||
}
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this schema is equal to another schema.
|
||||
/// </summary>
|
||||
/// <param name="other">The schema to compare with.</param>
|
||||
/// <returns><see langword="true"/> when schemas are equal; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Equals(BsonSchema? other)
|
||||
{
|
||||
if (other == null) return false;
|
||||
return GetHash() == other.GetHash();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool Equals(object? obj) => Equals(obj as BsonSchema);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int GetHashCode() => (int)GetHash();
|
||||
|
||||
/// <summary>
|
||||
/// Enumerates all field keys in this schema, including nested schema keys.
|
||||
/// </summary>
|
||||
/// <returns>An enumerable of field keys.</returns>
|
||||
public IEnumerable<string> GetAllKeys()
|
||||
{
|
||||
foreach (var field in Fields)
|
||||
{
|
||||
yield return field.Name;
|
||||
if (field.NestedSchema != null)
|
||||
{
|
||||
foreach (var nestedKey in field.NestedSchema.GetAllKeys())
|
||||
{
|
||||
yield return nestedKey;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
28
src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj
Executable file
28
src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj
Executable file
@@ -0,0 +1,28 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD.Bson</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD.Bson</RootNamespace>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
|
||||
<PackageId>ZB.MOM.WW.CBDD.Bson</PackageId>
|
||||
<Version>1.3.1</Version>
|
||||
<Authors>CBDD Team</Authors>
|
||||
<Description>BSON Serialization Library for High-Performance Database Engine</Description>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
23
src/CBDD.CheckpointTest/DocumentDb.CheckpointTest.csproj
Executable file
23
src/CBDD.CheckpointTest/DocumentDb.CheckpointTest.csproj
Executable file
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD.CheckpointTest</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD.CheckpointTest</RootNamespace>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>14.0</LangVersion>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.10" />
|
||||
<PackageReference Include="Serilog" Version="4.2.0" />
|
||||
<PackageReference Include="Serilog.Extensions.Logging" Version="8.0.0" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
|
||||
<ProjectReference Include="..\CBDD.Core\ZB.MOM.WW.CBDD.Core.csproj" />
|
||||
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
149
src/CBDD.CheckpointTest/Program.cs
Executable file
149
src/CBDD.CheckpointTest/Program.cs
Executable file
@@ -0,0 +1,149 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Serilog;
|
||||
using Serilog.Context;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.CheckpointTest;
|
||||
|
||||
/// <summary>
|
||||
/// Quick test to verify checkpoint functionality and performance improvement
|
||||
/// </summary>
|
||||
class Program
|
||||
{
|
||||
static void Main(string[] args)
|
||||
{
|
||||
var serilogLogger = new LoggerConfiguration()
|
||||
.Enrich.FromLogContext()
|
||||
.WriteTo.Console()
|
||||
.CreateLogger();
|
||||
|
||||
using var loggerFactory = LoggerFactory.Create(builder =>
|
||||
{
|
||||
builder.ClearProviders();
|
||||
builder.AddSerilog(serilogLogger, dispose: true);
|
||||
});
|
||||
|
||||
var logger = loggerFactory.CreateLogger<Program>();
|
||||
logger.LogInformation("=== DocumentDb Checkpoint Performance Test ===");
|
||||
|
||||
var dbPath = "test_checkpoint.db";
|
||||
var walPath = "test_checkpoint.wal";
|
||||
|
||||
using var _ = LogContext.PushProperty("DatabasePath", dbPath);
|
||||
using var __ = LogContext.PushProperty("WalPath", walPath);
|
||||
|
||||
if (File.Exists(dbPath))
|
||||
{
|
||||
File.Delete(dbPath);
|
||||
}
|
||||
|
||||
if (File.Exists(walPath))
|
||||
{
|
||||
File.Delete(walPath);
|
||||
}
|
||||
|
||||
using var storage = new StorageEngine(dbPath, PageFileConfig.Default);
|
||||
|
||||
logger.LogInformation("1. Testing Single Inserts ({TransactionCount} transactions)...", 500);
|
||||
var sw = Stopwatch.StartNew();
|
||||
|
||||
using (LogContext.PushProperty("Phase", "SingleInserts"))
|
||||
{
|
||||
for (int i = 0; i < 500; i++)
|
||||
{
|
||||
using var txn = storage.BeginTransaction();
|
||||
|
||||
// Simulate a write
|
||||
var pageId = storage.AllocatePage();
|
||||
var data = new byte[storage.PageSize];
|
||||
new Random().NextBytes(data);
|
||||
|
||||
// Write directly to storage
|
||||
// Old: txn.AddWrite(...)
|
||||
storage.WritePage(pageId, txn.TransactionId, data);
|
||||
|
||||
storage.CommitTransaction(txn);
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
logger.LogInformation("Completed {TransactionCount} inserts in {ElapsedMs}ms", 500, sw.ElapsedMilliseconds);
|
||||
logger.LogInformation("Average: {InsertsPerSecond:F0} inserts/sec", 500.0 / sw.Elapsed.TotalSeconds);
|
||||
|
||||
var walSize = new FileInfo(walPath).Length;
|
||||
logger.LogInformation("WAL size: {WalSizeKb:F1} KB", walSize / 1024.0);
|
||||
|
||||
logger.LogInformation("2. Performing Manual Checkpoint...");
|
||||
sw.Restart();
|
||||
storage.Checkpoint();
|
||||
sw.Stop();
|
||||
|
||||
logger.LogInformation("Checkpoint completed in {ElapsedMs}ms", sw.ElapsedMilliseconds);
|
||||
|
||||
var dbSize = new FileInfo(dbPath).Length;
|
||||
var walSizeAfter = new FileInfo(walPath).Length;
|
||||
logger.LogInformation("DB size: {DbSizeKb:F1} KB", dbSize / 1024.0);
|
||||
logger.LogInformation("WAL size after checkpoint: {WalSizeKb:F1} KB", walSizeAfter / 1024.0);
|
||||
|
||||
logger.LogInformation("3. Testing Checkpoint with Truncate (Integrated)...");
|
||||
storage.Checkpoint();
|
||||
|
||||
walSizeAfter = new FileInfo(walPath).Length;
|
||||
logger.LogInformation("WAL size after truncate: {WalSizeKb:F1} KB", walSizeAfter / 1024.0);
|
||||
|
||||
logger.LogInformation("4. Testing Batch Inserts ({TransactionCount} transactions)...", 1000);
|
||||
sw.Restart();
|
||||
|
||||
using (LogContext.PushProperty("Phase", "BatchInserts"))
|
||||
{
|
||||
for (int i = 0; i < 1000; i++)
|
||||
{
|
||||
using var txn = storage.BeginTransaction();
|
||||
|
||||
var pageId = storage.AllocatePage();
|
||||
var data = new byte[storage.PageSize];
|
||||
new Random().NextBytes(data);
|
||||
|
||||
storage.WritePage(pageId, txn.TransactionId, data);
|
||||
|
||||
storage.CommitTransaction(txn);
|
||||
}
|
||||
}
|
||||
|
||||
sw.Stop();
|
||||
logger.LogInformation("Completed {TransactionCount} inserts in {ElapsedMs}ms", 1000, sw.ElapsedMilliseconds);
|
||||
logger.LogInformation("Average: {InsertsPerSecond:F0} inserts/sec", 1000.0 / sw.Elapsed.TotalSeconds);
|
||||
|
||||
walSize = new FileInfo(walPath).Length;
|
||||
logger.LogInformation("WAL size: {WalSizeKb:F1} KB", walSize / 1024.0);
|
||||
|
||||
logger.LogInformation("5. Final checkpoint and cleanup...");
|
||||
storage.Checkpoint();
|
||||
|
||||
dbSize = new FileInfo(dbPath).Length;
|
||||
walSizeAfter = new FileInfo(walPath).Length;
|
||||
logger.LogInformation("Final DB size: {DbSizeKb:F1} KB", dbSize / 1024.0);
|
||||
logger.LogInformation("Final WAL size: {WalSizeKb:F1} KB", walSizeAfter / 1024.0);
|
||||
|
||||
logger.LogInformation("=== Test Completed Successfully! ===");
|
||||
logger.LogInformation("Key Observations:");
|
||||
logger.LogInformation("- Commits are fast (only WAL writes)");
|
||||
logger.LogInformation("- Checkpoint consolidates changes to DB");
|
||||
logger.LogInformation("- Truncate reclaims WAL space");
|
||||
logger.LogInformation("Press any key to exit...");
|
||||
Console.ReadKey();
|
||||
|
||||
// Cleanup
|
||||
try
|
||||
{
|
||||
if (File.Exists(dbPath)) File.Delete(dbPath);
|
||||
if (File.Exists(walPath)) File.Delete(walPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogWarning(ex, "Cleanup after checkpoint test failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
155
src/CBDD.Core/CDC/ChangeStreamDispatcher.cs
Executable file
155
src/CBDD.Core/CDC/ChangeStreamDispatcher.cs
Executable file
@@ -0,0 +1,155 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Channels;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||
|
||||
internal sealed class ChangeStreamDispatcher : IDisposable
|
||||
{
|
||||
private readonly Channel<InternalChangeEvent> _channel;
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>> _subscriptions = new();
|
||||
private readonly ConcurrentDictionary<string, int> _payloadWatcherCounts = new();
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new change stream dispatcher.
|
||||
/// </summary>
|
||||
public ChangeStreamDispatcher()
|
||||
{
|
||||
_channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = false
|
||||
});
|
||||
|
||||
Task.Run(ProcessEventsAsync);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes a change event to subscribers.
|
||||
/// </summary>
|
||||
/// <param name="change">The change event to publish.</param>
|
||||
public void Publish(InternalChangeEvent change)
|
||||
{
|
||||
_channel.Writer.TryWrite(change);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a collection has subscribers that require payloads.
|
||||
/// </summary>
|
||||
/// <param name="collectionName">The collection name.</param>
|
||||
/// <returns><see langword="true"/> if payload watchers exist; otherwise, <see langword="false"/>.</returns>
|
||||
public bool HasPayloadWatchers(string collectionName)
|
||||
{
|
||||
return _payloadWatcherCounts.TryGetValue(collectionName, out var count) && count > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a collection has any subscribers.
|
||||
/// </summary>
|
||||
/// <param name="collectionName">The collection name.</param>
|
||||
/// <returns><see langword="true"/> if subscribers exist; otherwise, <see langword="false"/>.</returns>
|
||||
public bool HasAnyWatchers(string collectionName)
|
||||
{
|
||||
return _subscriptions.TryGetValue(collectionName, out var subs) && !subs.IsEmpty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Subscribes a channel writer to collection change events.
|
||||
/// </summary>
|
||||
/// <param name="collectionName">The collection name to subscribe to.</param>
|
||||
/// <param name="capturePayload">Whether this subscriber requires event payloads.</param>
|
||||
/// <param name="writer">The destination channel writer.</param>
|
||||
/// <returns>An <see cref="IDisposable"/> that removes the subscription when disposed.</returns>
|
||||
public IDisposable Subscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
||||
{
|
||||
if (capturePayload)
|
||||
{
|
||||
_payloadWatcherCounts.AddOrUpdate(collectionName, 1, (_, count) => count + 1);
|
||||
}
|
||||
|
||||
var collectionSubs = _subscriptions.GetOrAdd(collectionName, _ => new ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>());
|
||||
collectionSubs.TryAdd(writer, 0);
|
||||
|
||||
return new Subscription(() => Unsubscribe(collectionName, capturePayload, writer));
|
||||
}
|
||||
|
||||
private void Unsubscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
||||
{
|
||||
if (_subscriptions.TryGetValue(collectionName, out var collectionSubs))
|
||||
{
|
||||
collectionSubs.TryRemove(writer, out _);
|
||||
}
|
||||
|
||||
if (capturePayload)
|
||||
{
|
||||
_payloadWatcherCounts.AddOrUpdate(collectionName, 0, (_, count) => Math.Max(0, count - 1));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessEventsAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var reader = _channel.Reader;
|
||||
while (await reader.WaitToReadAsync(_cts.Token))
|
||||
{
|
||||
while (reader.TryRead(out var @event))
|
||||
{
|
||||
if (_subscriptions.TryGetValue(@event.CollectionName, out var collectionSubs))
|
||||
{
|
||||
foreach (var writer in collectionSubs.Keys)
|
||||
{
|
||||
// Optimized fan-out: non-blocking TryWrite.
|
||||
// If a subscriber channel is full (unlikely with Unbounded),
|
||||
// we skip or drop. Usually, subscribers will also use Unbounded.
|
||||
writer.TryWrite(@event);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { }
|
||||
catch (Exception)
|
||||
{
|
||||
// Internal error logging could go here
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases dispatcher resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
}
|
||||
|
||||
private sealed class Subscription : IDisposable
|
||||
{
|
||||
private readonly Action _onDispose;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new subscription token.
|
||||
/// </summary>
|
||||
/// <param name="onDispose">Callback executed when the subscription is disposed.</param>
|
||||
public Subscription(Action onDispose)
|
||||
{
|
||||
_onDispose = onDispose;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the subscription and unregisters the subscriber.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_onDispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
76
src/CBDD.Core/CDC/ChangeStreamEvent.cs
Executable file
76
src/CBDD.Core/CDC/ChangeStreamEvent.cs
Executable file
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||
|
||||
/// <summary>
|
||||
/// A generic, immutable struct representing a data change in a collection.
|
||||
/// </summary>
|
||||
public readonly struct ChangeStreamEvent<TId, T> where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the UTC timestamp when the change was recorded.
|
||||
/// </summary>
|
||||
public long Timestamp { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the transaction identifier that produced the change.
|
||||
/// </summary>
|
||||
public ulong TransactionId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the collection name where the change occurred.
|
||||
/// </summary>
|
||||
public string CollectionName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the operation type associated with the change.
|
||||
/// </summary>
|
||||
public OperationType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the changed document identifier.
|
||||
/// </summary>
|
||||
public TId DocumentId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The deserialized entity. Null if capturePayload was false during Watch().
|
||||
/// </summary>
|
||||
public T? Entity { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Low-level event structure used internally to transport changes before deserialization.
|
||||
/// </summary>
|
||||
internal readonly struct InternalChangeEvent
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the UTC timestamp when the change was recorded.
|
||||
/// </summary>
|
||||
public long Timestamp { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the transaction identifier that produced the change.
|
||||
/// </summary>
|
||||
public ulong TransactionId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the collection name where the change occurred.
|
||||
/// </summary>
|
||||
public string CollectionName { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the operation type associated with the change.
|
||||
/// </summary>
|
||||
public OperationType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Raw BSON of the Document ID.
|
||||
/// </summary>
|
||||
public ReadOnlyMemory<byte> IdBytes { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Raw BSON of the Entity. Null if payload not captured.
|
||||
/// </summary>
|
||||
public ReadOnlyMemory<byte>? PayloadBytes { get; init; }
|
||||
}
|
||||
150
src/CBDD.Core/CDC/ChangeStreamObservable.cs
Executable file
150
src/CBDD.Core/CDC/ChangeStreamObservable.cs
Executable file
@@ -0,0 +1,150 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Channels;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||
|
||||
internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamEvent<TId, T>> where T : class
|
||||
{
|
||||
private readonly ChangeStreamDispatcher _dispatcher;
|
||||
private readonly string _collectionName;
|
||||
private readonly bool _capturePayload;
|
||||
private readonly IDocumentMapper<TId, T> _mapper;
|
||||
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new observable wrapper for collection change events.
|
||||
/// </summary>
|
||||
/// <param name="dispatcher">The dispatcher producing internal change events.</param>
|
||||
/// <param name="collectionName">The collection to subscribe to.</param>
|
||||
/// <param name="capturePayload">Whether full entity payloads should be included.</param>
|
||||
/// <param name="mapper">The document mapper used for ID and payload deserialization.</param>
|
||||
/// <param name="keyReverseMap">The key reverse map used by BSON readers.</param>
|
||||
public ChangeStreamObservable(
|
||||
ChangeStreamDispatcher dispatcher,
|
||||
string collectionName,
|
||||
bool capturePayload,
|
||||
IDocumentMapper<TId, T> mapper,
|
||||
ConcurrentDictionary<ushort, string> keyReverseMap)
|
||||
{
|
||||
_dispatcher = dispatcher;
|
||||
_collectionName = collectionName;
|
||||
_capturePayload = capturePayload;
|
||||
_mapper = mapper;
|
||||
_keyReverseMap = keyReverseMap;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDisposable Subscribe(IObserver<ChangeStreamEvent<TId, T>> observer)
|
||||
{
|
||||
if (observer == null) throw new ArgumentNullException(nameof(observer));
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = true
|
||||
});
|
||||
|
||||
var dispatcherSubscription = _dispatcher.Subscribe(_collectionName, _capturePayload, channel.Writer);
|
||||
|
||||
// Background task to bridge Channel -> Observer
|
||||
var bridgeTask = Task.Run(() => BridgeChannelToObserverAsync(channel.Reader, observer, cts.Token));
|
||||
|
||||
return new CompositeDisposable(dispatcherSubscription, cts, channel.Writer, bridgeTask);
|
||||
}
|
||||
|
||||
private async Task BridgeChannelToObserverAsync(ChannelReader<InternalChangeEvent> reader, IObserver<ChangeStreamEvent<TId, T>> observer, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (await reader.WaitToReadAsync(ct))
|
||||
{
|
||||
while (reader.TryRead(out var internalEvent))
|
||||
{
|
||||
try
|
||||
{
|
||||
// Deserializza ID
|
||||
var eventId = _mapper.FromIndexKey(new IndexKey(internalEvent.IdBytes.ToArray()));
|
||||
|
||||
// Deserializza Payload (se presente)
|
||||
T? entity = default;
|
||||
if (internalEvent.PayloadBytes.HasValue)
|
||||
{
|
||||
entity = _mapper.Deserialize(new BsonSpanReader(internalEvent.PayloadBytes.Value.Span, _keyReverseMap));
|
||||
}
|
||||
|
||||
var externalEvent = new ChangeStreamEvent<TId, T>
|
||||
{
|
||||
Timestamp = internalEvent.Timestamp,
|
||||
TransactionId = internalEvent.TransactionId,
|
||||
CollectionName = internalEvent.CollectionName,
|
||||
Type = internalEvent.Type,
|
||||
DocumentId = eventId,
|
||||
Entity = entity
|
||||
};
|
||||
|
||||
observer.OnNext(externalEvent);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// In case of deserialization error, we notify and continue if possible
|
||||
// Or we can stop the observer.
|
||||
observer.OnError(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
observer.OnCompleted();
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
observer.OnCompleted();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
observer.OnError(ex);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CompositeDisposable : IDisposable
|
||||
{
|
||||
private readonly IDisposable _dispatcherSubscription;
|
||||
private readonly CancellationTokenSource _cts;
|
||||
private readonly ChannelWriter<InternalChangeEvent> _writer;
|
||||
private readonly Task _bridgeTask;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new disposable wrapper for change stream resources.
|
||||
/// </summary>
|
||||
/// <param name="dispatcherSubscription">The dispatcher subscription handle.</param>
|
||||
/// <param name="cts">The cancellation source controlling the bridge task.</param>
|
||||
/// <param name="writer">The channel writer for internal change events.</param>
|
||||
/// <param name="bridgeTask">The running bridge task.</param>
|
||||
public CompositeDisposable(IDisposable dispatcherSubscription, CancellationTokenSource cts, ChannelWriter<InternalChangeEvent> writer, Task bridgeTask)
|
||||
{
|
||||
_dispatcherSubscription = dispatcherSubscription;
|
||||
_cts = cts;
|
||||
_writer = writer;
|
||||
_bridgeTask = bridgeTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_disposed = true;
|
||||
|
||||
_dispatcherSubscription.Dispose();
|
||||
_writer.TryComplete();
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
79
src/CBDD.Core/CDC/CollectionCdcPublisher.cs
Normal file
79
src/CBDD.Core/CDC/CollectionCdcPublisher.cs
Normal file
@@ -0,0 +1,79 @@
|
||||
using System.Collections.Concurrent;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||
|
||||
/// <summary>
|
||||
/// Handles CDC watch/notify behavior for a single collection.
|
||||
/// Extracted from DocumentCollection to keep storage/query concerns separated from event plumbing.
|
||||
/// </summary>
|
||||
/// <typeparam name="TId">Document identifier type.</typeparam>
|
||||
/// <typeparam name="T">Document type.</typeparam>
|
||||
internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
||||
{
|
||||
private readonly ITransactionHolder _transactionHolder;
|
||||
private readonly string _collectionName;
|
||||
private readonly IDocumentMapper<TId, T> _mapper;
|
||||
private readonly ChangeStreamDispatcher? _dispatcher;
|
||||
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
||||
|
||||
public CollectionCdcPublisher(
|
||||
ITransactionHolder transactionHolder,
|
||||
string collectionName,
|
||||
IDocumentMapper<TId, T> mapper,
|
||||
ChangeStreamDispatcher? dispatcher,
|
||||
ConcurrentDictionary<ushort, string> keyReverseMap)
|
||||
{
|
||||
_transactionHolder = transactionHolder ?? throw new ArgumentNullException(nameof(transactionHolder));
|
||||
_collectionName = collectionName ?? throw new ArgumentNullException(nameof(collectionName));
|
||||
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
||||
_dispatcher = dispatcher;
|
||||
_keyReverseMap = keyReverseMap ?? throw new ArgumentNullException(nameof(keyReverseMap));
|
||||
}
|
||||
|
||||
public IObservable<ChangeStreamEvent<TId, T>> Watch(bool capturePayload = false)
|
||||
{
|
||||
if (_dispatcher == null)
|
||||
throw new InvalidOperationException("CDC is not initialized.");
|
||||
|
||||
return new ChangeStreamObservable<TId, T>(
|
||||
_dispatcher,
|
||||
_collectionName,
|
||||
capturePayload,
|
||||
_mapper,
|
||||
_keyReverseMap);
|
||||
}
|
||||
|
||||
public void Notify(OperationType type, TId id, ReadOnlySpan<byte> docData = default)
|
||||
{
|
||||
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
||||
if (_dispatcher == null)
|
||||
return;
|
||||
|
||||
if (!_dispatcher.HasAnyWatchers(_collectionName))
|
||||
return;
|
||||
|
||||
ReadOnlyMemory<byte>? payload = null;
|
||||
if (!docData.IsEmpty && _dispatcher.HasPayloadWatchers(_collectionName))
|
||||
{
|
||||
payload = docData.ToArray();
|
||||
}
|
||||
|
||||
var idBytes = _mapper.ToIndexKey(id).Data.ToArray();
|
||||
|
||||
if (transaction is Transaction t)
|
||||
{
|
||||
t.AddChange(new InternalChangeEvent
|
||||
{
|
||||
Timestamp = DateTime.UtcNow.Ticks,
|
||||
TransactionId = transaction.TransactionId,
|
||||
CollectionName = _collectionName,
|
||||
Type = type,
|
||||
IdBytes = idBytes,
|
||||
PayloadBytes = payload
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
122
src/CBDD.Core/Collections/BaseMappers.cs
Executable file
122
src/CBDD.Core/Collections/BaseMappers.cs
Executable file
@@ -0,0 +1,122 @@
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for custom mappers that provides bidirectional IndexKey mapping for standard types.
|
||||
/// </summary>
|
||||
public abstract class DocumentMapperBase<TId, T> : IDocumentMapper<TId, T> where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the target collection name for the mapped entity type.
|
||||
/// </summary>
|
||||
public abstract string CollectionName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Serializes an entity instance into BSON.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity to serialize.</param>
|
||||
/// <param name="writer">The BSON writer to write into.</param>
|
||||
/// <returns>The number of bytes written.</returns>
|
||||
public abstract int Serialize(T entity, BsonSpanWriter writer);
|
||||
|
||||
/// <summary>
|
||||
/// Deserializes an entity instance from BSON.
|
||||
/// </summary>
|
||||
/// <param name="reader">The BSON reader to read from.</param>
|
||||
/// <returns>The deserialized entity.</returns>
|
||||
public abstract T Deserialize(BsonSpanReader reader);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the identifier value from an entity.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity to read the identifier from.</param>
|
||||
/// <returns>The identifier value.</returns>
|
||||
public abstract TId GetId(T entity);
|
||||
|
||||
/// <summary>
|
||||
/// Sets the identifier value on an entity.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity to update.</param>
|
||||
/// <param name="id">The identifier value to assign.</param>
|
||||
public abstract void SetId(T entity, TId id);
|
||||
|
||||
/// <summary>
|
||||
/// Converts a typed identifier value into an index key.
|
||||
/// </summary>
|
||||
/// <param name="id">The identifier value.</param>
|
||||
/// <returns>The index key representation of the identifier.</returns>
|
||||
public virtual IndexKey ToIndexKey(TId id) => IndexKey.Create(id);
|
||||
|
||||
/// <summary>
|
||||
/// Converts an index key back into a typed identifier value.
|
||||
/// </summary>
|
||||
/// <param name="key">The index key to convert.</param>
|
||||
/// <returns>The typed identifier value.</returns>
|
||||
public virtual TId FromIndexKey(IndexKey key) => key.As<TId>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets all mapped field keys used by this mapper.
|
||||
/// </summary>
|
||||
public virtual IEnumerable<string> UsedKeys => GetSchema().GetAllKeys();
|
||||
|
||||
/// <summary>
|
||||
/// Builds the BSON schema for the mapped entity type.
|
||||
/// </summary>
|
||||
/// <returns>The generated BSON schema.</returns>
|
||||
public virtual BsonSchema GetSchema() => BsonSchemaGenerator.FromType<T>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base class for mappers using ObjectId as primary key.
|
||||
/// </summary>
|
||||
public abstract class ObjectIdMapperBase<T> : DocumentMapperBase<ObjectId, T>, IDocumentMapper<T> where T : class
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override IndexKey ToIndexKey(ObjectId id) => IndexKey.Create(id);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ObjectId FromIndexKey(IndexKey key) => key.As<ObjectId>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base class for mappers using Int32 as primary key.
|
||||
/// </summary>
|
||||
public abstract class Int32MapperBase<T> : DocumentMapperBase<int, T> where T : class
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override IndexKey ToIndexKey(int id) => IndexKey.Create(id);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int FromIndexKey(IndexKey key) => key.As<int>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base class for mappers using String as primary key.
|
||||
/// </summary>
|
||||
public abstract class StringMapperBase<T> : DocumentMapperBase<string, T> where T : class
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override IndexKey ToIndexKey(string id) => IndexKey.Create(id);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string FromIndexKey(IndexKey key) => key.As<string>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Base class for mappers using Guid as primary key.
|
||||
/// </summary>
|
||||
public abstract class GuidMapperBase<T> : DocumentMapperBase<Guid, T> where T : class
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override IndexKey ToIndexKey(Guid id) => IndexKey.Create(id);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Guid FromIndexKey(IndexKey key) => key.As<Guid>();
|
||||
}
|
||||
138
src/CBDD.Core/Collections/BsonSchemaGenerator.cs
Executable file
138
src/CBDD.Core/Collections/BsonSchemaGenerator.cs
Executable file
@@ -0,0 +1,138 @@
|
||||
using System.Reflection;
|
||||
using System.Linq;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Concurrent;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using System;
|
||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
public static class BsonSchemaGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates a BSON schema for the specified CLR type.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The CLR type to inspect.</typeparam>
|
||||
/// <returns>The generated BSON schema.</returns>
|
||||
public static BsonSchema FromType<T>()
|
||||
{
|
||||
return FromType(typeof(T));
|
||||
}
|
||||
|
||||
private static readonly ConcurrentDictionary<Type, BsonSchema> _cache = new();
|
||||
|
||||
/// <summary>
|
||||
/// Generates a BSON schema for the specified CLR type.
|
||||
/// </summary>
|
||||
/// <param name="type">The CLR type to inspect.</param>
|
||||
/// <returns>The generated BSON schema.</returns>
|
||||
public static BsonSchema FromType(Type type)
|
||||
{
|
||||
return _cache.GetOrAdd(type, GenerateSchema);
|
||||
}
|
||||
|
||||
private static BsonSchema GenerateSchema(Type type)
|
||||
{
|
||||
var schema = new BsonSchema { Title = type.Name };
|
||||
var properties = type.GetProperties(BindingFlags.Public | BindingFlags.Instance);
|
||||
var fields = type.GetFields(BindingFlags.Public | BindingFlags.Instance);
|
||||
|
||||
foreach (var prop in properties)
|
||||
{
|
||||
if (prop.GetIndexParameters().Length > 0) continue; // Skip indexers
|
||||
if (!prop.CanRead) continue;
|
||||
|
||||
AddField(schema, prop.Name, prop.PropertyType);
|
||||
}
|
||||
|
||||
foreach (var field in fields)
|
||||
{
|
||||
AddField(schema, field.Name, field.FieldType);
|
||||
}
|
||||
|
||||
return schema;
|
||||
}
|
||||
|
||||
private static void AddField(BsonSchema schema, string name, Type type)
|
||||
{
|
||||
name = name.ToLowerInvariant();
|
||||
|
||||
// Convention: id -> _id for root document
|
||||
if (name.Equals("id", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
name = "_id";
|
||||
}
|
||||
|
||||
var (bsonType, nestedSchema, itemType) = GetBsonType(type);
|
||||
|
||||
schema.Fields.Add(new BsonField
|
||||
{
|
||||
Name = name,
|
||||
Type = bsonType,
|
||||
IsNullable = IsNullable(type),
|
||||
NestedSchema = nestedSchema,
|
||||
ArrayItemType = itemType
|
||||
});
|
||||
}
|
||||
|
||||
private static (BsonType type, BsonSchema? nested, BsonType? itemType) GetBsonType(Type type)
|
||||
{
|
||||
// Handle Nullable<T>
|
||||
type = Nullable.GetUnderlyingType(type) ?? type;
|
||||
|
||||
if (type == typeof(ObjectId)) return (BsonType.ObjectId, null, null);
|
||||
if (type == typeof(string)) return (BsonType.String, null, null);
|
||||
if (type == typeof(int)) return (BsonType.Int32, null, null);
|
||||
if (type == typeof(long)) return (BsonType.Int64, null, null);
|
||||
if (type == typeof(bool)) return (BsonType.Boolean, null, null);
|
||||
if (type == typeof(double)) return (BsonType.Double, null, null);
|
||||
if (type == typeof(decimal)) return (BsonType.Decimal128, null, null);
|
||||
if (type == typeof(DateTime) || type == typeof(DateTimeOffset)) return (BsonType.DateTime, null, null);
|
||||
if (type == typeof(Guid)) return (BsonType.Binary, null, null); // Guid is usually Binary subtype
|
||||
if (type == typeof(byte[])) return (BsonType.Binary, null, null);
|
||||
|
||||
// Arrays/Lists
|
||||
if (type != typeof(string) && typeof(IEnumerable).IsAssignableFrom(type))
|
||||
{
|
||||
var itemType = GetCollectionItemType(type);
|
||||
var (itemBsonType, itemNested, _) = GetBsonType(itemType);
|
||||
|
||||
// For arrays, if item is Document, we use NestedSchema to describe the item
|
||||
return (BsonType.Array, itemNested, itemBsonType);
|
||||
}
|
||||
|
||||
// Nested Objects / Structs
|
||||
// If it's not a string, not a primitive, and not an array/list, treat as Document
|
||||
if (type != typeof(string) && !type.IsPrimitive && !type.IsEnum)
|
||||
{
|
||||
// Avoid infinite recursion?
|
||||
// Simple approach: generating nested schema
|
||||
return (BsonType.Document, FromType(type), null);
|
||||
}
|
||||
|
||||
return (BsonType.Undefined, null, null);
|
||||
}
|
||||
|
||||
private static bool IsNullable(Type type)
|
||||
{
|
||||
return !type.IsValueType || Nullable.GetUnderlyingType(type) != null;
|
||||
}
|
||||
|
||||
private static Type GetCollectionItemType(Type type)
|
||||
{
|
||||
if (type.IsArray) return type.GetElementType()!;
|
||||
|
||||
// If type itself is IEnumerable<T>
|
||||
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>))
|
||||
{
|
||||
return type.GetGenericArguments()[0];
|
||||
}
|
||||
|
||||
var enumerableType = type.GetInterfaces()
|
||||
.FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>));
|
||||
|
||||
return enumerableType?.GetGenericArguments()[0] ?? typeof(object);
|
||||
}
|
||||
}
|
||||
101
src/CBDD.Core/Collections/DocumentCollection.Scan.cs
Normal file
101
src/CBDD.Core/Collections/DocumentCollection.Scan.cs
Normal file
@@ -0,0 +1,101 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.InteropServices;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
public partial class DocumentCollection<TId, T> where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Scans the entire collection using a raw BSON predicate.
|
||||
/// This avoids deserializing documents that don't match the criteria.
|
||||
/// </summary>
|
||||
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
||||
/// <returns>Matching documents</returns>
|
||||
internal IEnumerable<T> Scan(Func<BsonSpanReader, bool> predicate)
|
||||
{
|
||||
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
||||
|
||||
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
||||
var txnId = transaction.TransactionId;
|
||||
var pageCount = _storage.PageCount;
|
||||
var buffer = new byte[_storage.PageSize];
|
||||
var pageResults = new List<T>();
|
||||
|
||||
for (uint pageId = 0; pageId < pageCount; pageId++)
|
||||
{
|
||||
pageResults.Clear();
|
||||
ScanPage(pageId, txnId, buffer, predicate, pageResults);
|
||||
|
||||
foreach (var doc in pageResults)
|
||||
{
|
||||
yield return doc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scans the collection in parallel using multiple threads.
|
||||
/// Useful for large collections on multi-core machines.
|
||||
/// </summary>
|
||||
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
||||
/// <param name="degreeOfParallelism">Number of threads to use (default: -1 = ProcessorCount)</param>
|
||||
internal IEnumerable<T> ParallelScan(Func<BsonSpanReader, bool> predicate, int degreeOfParallelism = -1)
|
||||
{
|
||||
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
||||
|
||||
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
||||
var txnId = transaction.TransactionId;
|
||||
var pageCount = (int)_storage.PageCount;
|
||||
|
||||
if (degreeOfParallelism <= 0)
|
||||
degreeOfParallelism = Environment.ProcessorCount;
|
||||
|
||||
return Partitioner.Create(0, pageCount)
|
||||
.AsParallel()
|
||||
.WithDegreeOfParallelism(degreeOfParallelism)
|
||||
.SelectMany(range =>
|
||||
{
|
||||
var localBuffer = new byte[_storage.PageSize];
|
||||
var localResults = new List<T>();
|
||||
|
||||
for (int i = range.Item1; i < range.Item2; i++)
|
||||
{
|
||||
ScanPage((uint)i, txnId, localBuffer, predicate, localResults);
|
||||
}
|
||||
|
||||
return localResults;
|
||||
});
|
||||
}
|
||||
|
||||
private void ScanPage(uint pageId, ulong txnId, byte[] buffer, Func<BsonSpanReader, bool> predicate, List<T> results)
|
||||
{
|
||||
_storage.ReadPage(pageId, txnId, buffer);
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
|
||||
if (header.PageType != PageType.Data)
|
||||
return;
|
||||
|
||||
var slots = MemoryMarshal.Cast<byte, SlotEntry>(
|
||||
buffer.AsSpan(SlottedPageHeader.Size, header.SlotCount * SlotEntry.Size));
|
||||
|
||||
for (int i = 0; i < header.SlotCount; i++)
|
||||
{
|
||||
var slot = slots[i];
|
||||
|
||||
if (slot.Flags.HasFlag(SlotFlags.Deleted))
|
||||
continue;
|
||||
|
||||
var data = buffer.AsSpan(slot.Offset, slot.Length);
|
||||
var reader = new BsonSpanReader(data, _storage.GetKeyReverseMap());
|
||||
|
||||
if (predicate(reader))
|
||||
{
|
||||
var doc = FindByLocation(new DocumentLocation(pageId, (ushort)i));
|
||||
if (doc != null)
|
||||
results.Add(doc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
1992
src/CBDD.Core/Collections/DocumentCollection.cs
Executable file
1992
src/CBDD.Core/Collections/DocumentCollection.cs
Executable file
File diff suppressed because it is too large
Load Diff
87
src/CBDD.Core/Collections/IDocumentMapper.cs
Executable file
87
src/CBDD.Core/Collections/IDocumentMapper.cs
Executable file
@@ -0,0 +1,87 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
/// <summary>
|
||||
/// Non-generic interface for common mapper operations.
|
||||
/// </summary>
|
||||
public interface IDocumentMapper
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the collection name handled by this mapper.
|
||||
/// </summary>
|
||||
string CollectionName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the set of document keys used during mapping.
|
||||
/// </summary>
|
||||
IEnumerable<string> UsedKeys { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the BSON schema for the mapped document.
|
||||
/// </summary>
|
||||
/// <returns>The BSON schema.</returns>
|
||||
BsonSchema GetSchema();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface for mapping between entities and BSON using zero-allocation serialization.
|
||||
/// Handles bidirectional mapping between TId and IndexKey.
|
||||
/// </summary>
|
||||
public interface IDocumentMapper<TId, T> : IDocumentMapper where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Serializes an entity to BSON.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity to serialize.</param>
|
||||
/// <param name="writer">The BSON writer.</param>
|
||||
/// <returns>The number of bytes written.</returns>
|
||||
int Serialize(T entity, BsonSpanWriter writer);
|
||||
|
||||
/// <summary>
|
||||
/// Deserializes an entity from BSON.
|
||||
/// </summary>
|
||||
/// <param name="reader">The BSON reader.</param>
|
||||
/// <returns>The deserialized entity.</returns>
|
||||
T Deserialize(BsonSpanReader reader);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the identifier value from an entity.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity.</param>
|
||||
/// <returns>The identifier value.</returns>
|
||||
TId GetId(T entity);
|
||||
|
||||
/// <summary>
|
||||
/// Sets the identifier value on an entity.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity.</param>
|
||||
/// <param name="id">The identifier value.</param>
|
||||
void SetId(T entity, TId id);
|
||||
|
||||
/// <summary>
|
||||
/// Converts an identifier to an index key.
|
||||
/// </summary>
|
||||
/// <param name="id">The identifier value.</param>
|
||||
/// <returns>The index key representation.</returns>
|
||||
IndexKey ToIndexKey(TId id);
|
||||
|
||||
/// <summary>
|
||||
/// Converts an index key back to an identifier.
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <returns>The identifier value.</returns>
|
||||
TId FromIndexKey(IndexKey key);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Legacy interface for compatibility with existing ObjectId-based collections.
|
||||
/// </summary>
|
||||
public interface IDocumentMapper<T> : IDocumentMapper<ObjectId, T> where T : class
|
||||
{
|
||||
}
|
||||
30
src/CBDD.Core/Collections/SchemaVersion.cs
Executable file
30
src/CBDD.Core/Collections/SchemaVersion.cs
Executable file
@@ -0,0 +1,30 @@
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
public readonly struct SchemaVersion
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the schema version number.
|
||||
/// </summary>
|
||||
public int Version { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the schema hash.
|
||||
/// </summary>
|
||||
public long Hash { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SchemaVersion"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="version">The schema version number.</param>
|
||||
/// <param name="hash">The schema hash.</param>
|
||||
public SchemaVersion(int version, long hash)
|
||||
{
|
||||
Version = version;
|
||||
Hash = hash;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString() => $"v{Version} (0x{Hash:X16})";
|
||||
}
|
||||
118
src/CBDD.Core/Compression/CompressedPayloadHeader.cs
Normal file
118
src/CBDD.Core/Compression/CompressedPayloadHeader.cs
Normal file
@@ -0,0 +1,118 @@
|
||||
using System.Buffers.Binary;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Fixed header prefix for compressed payload blobs.
|
||||
/// </summary>
|
||||
public readonly struct CompressedPayloadHeader
|
||||
{
|
||||
public const int Size = 16;
|
||||
|
||||
/// <summary>
|
||||
/// Compression codec used for payload bytes.
|
||||
/// </summary>
|
||||
public CompressionCodec Codec { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Original uncompressed payload length.
|
||||
/// </summary>
|
||||
public int OriginalLength { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Compressed payload length.
|
||||
/// </summary>
|
||||
public int CompressedLength { get; }
|
||||
|
||||
/// <summary>
|
||||
/// CRC32 checksum of compressed payload bytes.
|
||||
/// </summary>
|
||||
public uint Checksum { get; }
|
||||
|
||||
public CompressedPayloadHeader(CompressionCodec codec, int originalLength, int compressedLength, uint checksum)
|
||||
{
|
||||
if (originalLength < 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(originalLength));
|
||||
if (compressedLength < 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(compressedLength));
|
||||
|
||||
Codec = codec;
|
||||
OriginalLength = originalLength;
|
||||
CompressedLength = compressedLength;
|
||||
Checksum = checksum;
|
||||
}
|
||||
|
||||
public static CompressedPayloadHeader Create(CompressionCodec codec, int originalLength, ReadOnlySpan<byte> compressedPayload)
|
||||
{
|
||||
var checksum = ComputeChecksum(compressedPayload);
|
||||
return new CompressedPayloadHeader(codec, originalLength, compressedPayload.Length, checksum);
|
||||
}
|
||||
|
||||
public void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < Size)
|
||||
throw new ArgumentException($"Destination must be at least {Size} bytes.", nameof(destination));
|
||||
|
||||
destination[0] = (byte)Codec;
|
||||
destination[1] = 0;
|
||||
destination[2] = 0;
|
||||
destination[3] = 0;
|
||||
BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(4, 4), OriginalLength);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(destination.Slice(8, 4), CompressedLength);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(destination.Slice(12, 4), Checksum);
|
||||
}
|
||||
|
||||
public static CompressedPayloadHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < Size)
|
||||
throw new ArgumentException($"Source must be at least {Size} bytes.", nameof(source));
|
||||
|
||||
var codec = (CompressionCodec)source[0];
|
||||
var originalLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(4, 4));
|
||||
var compressedLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(8, 4));
|
||||
var checksum = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(12, 4));
|
||||
return new CompressedPayloadHeader(codec, originalLength, compressedLength, checksum);
|
||||
}
|
||||
|
||||
public bool ValidateChecksum(ReadOnlySpan<byte> compressedPayload)
|
||||
{
|
||||
return Checksum == ComputeChecksum(compressedPayload);
|
||||
}
|
||||
|
||||
public static uint ComputeChecksum(ReadOnlySpan<byte> payload) => Crc32Calculator.Compute(payload);
|
||||
|
||||
private static class Crc32Calculator
|
||||
{
|
||||
private const uint Polynomial = 0xEDB88320u;
|
||||
private static readonly uint[] Table = CreateTable();
|
||||
|
||||
public static uint Compute(ReadOnlySpan<byte> payload)
|
||||
{
|
||||
uint crc = 0xFFFFFFFFu;
|
||||
for (int i = 0; i < payload.Length; i++)
|
||||
{
|
||||
var index = (crc ^ payload[i]) & 0xFF;
|
||||
crc = (crc >> 8) ^ Table[index];
|
||||
}
|
||||
|
||||
return ~crc;
|
||||
}
|
||||
|
||||
private static uint[] CreateTable()
|
||||
{
|
||||
var table = new uint[256];
|
||||
for (uint i = 0; i < table.Length; i++)
|
||||
{
|
||||
uint value = i;
|
||||
for (int bit = 0; bit < 8; bit++)
|
||||
{
|
||||
value = (value & 1) != 0 ? (value >> 1) ^ Polynomial : value >> 1;
|
||||
}
|
||||
|
||||
table[i] = value;
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
}
|
||||
}
|
||||
11
src/CBDD.Core/Compression/CompressionCodec.cs
Normal file
11
src/CBDD.Core/Compression/CompressionCodec.cs
Normal file
@@ -0,0 +1,11 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Supported payload compression codecs.
|
||||
/// </summary>
|
||||
public enum CompressionCodec : byte
|
||||
{
|
||||
None = 0,
|
||||
Brotli = 1,
|
||||
Deflate = 2
|
||||
}
|
||||
71
src/CBDD.Core/Compression/CompressionOptions.cs
Normal file
71
src/CBDD.Core/Compression/CompressionOptions.cs
Normal file
@@ -0,0 +1,71 @@
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Compression configuration for document payload processing.
|
||||
/// </summary>
|
||||
public sealed class CompressionOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Default compression options (compression disabled).
|
||||
/// </summary>
|
||||
public static CompressionOptions Default { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Enables payload compression for new writes.
|
||||
/// </summary>
|
||||
public bool EnableCompression { get; init; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum payload size (bytes) required before compression is attempted.
|
||||
/// </summary>
|
||||
public int MinSizeBytes { get; init; } = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum percentage of size reduction required to keep compressed output.
|
||||
/// </summary>
|
||||
public int MinSavingsPercent { get; init; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Preferred default codec for new writes.
|
||||
/// </summary>
|
||||
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
||||
|
||||
/// <summary>
|
||||
/// Compression level passed to codec implementations.
|
||||
/// </summary>
|
||||
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum allowed decompressed payload size.
|
||||
/// </summary>
|
||||
public int MaxDecompressedSizeBytes { get; init; } = 16 * 1024 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Optional maximum input size allowed for compression attempts.
|
||||
/// </summary>
|
||||
public int? MaxCompressionInputBytes { get; init; }
|
||||
|
||||
internal static CompressionOptions Normalize(CompressionOptions? options)
|
||||
{
|
||||
var candidate = options ?? Default;
|
||||
|
||||
if (candidate.MinSizeBytes < 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(MinSizeBytes), "MinSizeBytes must be non-negative.");
|
||||
|
||||
if (candidate.MinSavingsPercent is < 0 or > 100)
|
||||
throw new ArgumentOutOfRangeException(nameof(MinSavingsPercent), "MinSavingsPercent must be between 0 and 100.");
|
||||
|
||||
if (!Enum.IsDefined(candidate.Codec))
|
||||
throw new ArgumentOutOfRangeException(nameof(Codec), $"Unsupported codec: {candidate.Codec}.");
|
||||
|
||||
if (candidate.MaxDecompressedSizeBytes <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(MaxDecompressedSizeBytes), "MaxDecompressedSizeBytes must be greater than 0.");
|
||||
|
||||
if (candidate.MaxCompressionInputBytes is <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(MaxCompressionInputBytes), "MaxCompressionInputBytes must be greater than 0 when provided.");
|
||||
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
166
src/CBDD.Core/Compression/CompressionService.cs
Normal file
166
src/CBDD.Core/Compression/CompressionService.cs
Normal file
@@ -0,0 +1,166 @@
|
||||
using System.Buffers;
|
||||
using System.Collections.Concurrent;
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Compression codec registry and utility service.
|
||||
/// </summary>
|
||||
public sealed class CompressionService
|
||||
{
|
||||
private readonly ConcurrentDictionary<CompressionCodec, ICompressionCodec> _codecs = new();
|
||||
|
||||
public CompressionService(IEnumerable<ICompressionCodec>? additionalCodecs = null)
|
||||
{
|
||||
RegisterCodec(new NoneCompressionCodec());
|
||||
RegisterCodec(new BrotliCompressionCodec());
|
||||
RegisterCodec(new DeflateCompressionCodec());
|
||||
|
||||
if (additionalCodecs == null)
|
||||
return;
|
||||
|
||||
foreach (var codec in additionalCodecs)
|
||||
{
|
||||
RegisterCodec(codec);
|
||||
}
|
||||
}
|
||||
|
||||
public void RegisterCodec(ICompressionCodec codec)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(codec);
|
||||
_codecs[codec.Codec] = codec;
|
||||
}
|
||||
|
||||
public bool TryGetCodec(CompressionCodec codec, out ICompressionCodec compressionCodec)
|
||||
{
|
||||
return _codecs.TryGetValue(codec, out compressionCodec!);
|
||||
}
|
||||
|
||||
public ICompressionCodec GetCodec(CompressionCodec codec)
|
||||
{
|
||||
if (_codecs.TryGetValue(codec, out var compressionCodec))
|
||||
return compressionCodec;
|
||||
|
||||
throw new InvalidOperationException($"Compression codec '{codec}' is not registered.");
|
||||
}
|
||||
|
||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionCodec codec, CompressionLevel level)
|
||||
{
|
||||
return GetCodec(codec).Compress(input, level);
|
||||
}
|
||||
|
||||
public byte[] Decompress(ReadOnlySpan<byte> input, CompressionCodec codec, int expectedLength, int maxDecompressedSizeBytes)
|
||||
{
|
||||
return GetCodec(codec).Decompress(input, expectedLength, maxDecompressedSizeBytes);
|
||||
}
|
||||
|
||||
public byte[] Roundtrip(ReadOnlySpan<byte> input, CompressionCodec codec, CompressionLevel level, int maxDecompressedSizeBytes)
|
||||
{
|
||||
var compressed = Compress(input, codec, level);
|
||||
return Decompress(compressed, codec, input.Length, maxDecompressedSizeBytes);
|
||||
}
|
||||
|
||||
private sealed class NoneCompressionCodec : ICompressionCodec
|
||||
{
|
||||
public CompressionCodec Codec => CompressionCodec.None;
|
||||
|
||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level) => input.ToArray();
|
||||
|
||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||
{
|
||||
if (input.Length > maxDecompressedSizeBytes)
|
||||
throw new InvalidDataException($"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
||||
|
||||
if (expectedLength >= 0 && expectedLength != input.Length)
|
||||
throw new InvalidDataException($"Expected decompressed length {expectedLength}, actual {input.Length}.");
|
||||
|
||||
return input.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class BrotliCompressionCodec : ICompressionCodec
|
||||
{
|
||||
public CompressionCodec Codec => CompressionCodec.Brotli;
|
||||
|
||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
||||
{
|
||||
return CompressWithCodecStream(input, stream => new BrotliStream(stream, level, leaveOpen: true));
|
||||
}
|
||||
|
||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||
{
|
||||
return DecompressWithCodecStream(input, stream => new BrotliStream(stream, CompressionMode.Decompress, leaveOpen: true), expectedLength, maxDecompressedSizeBytes);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class DeflateCompressionCodec : ICompressionCodec
|
||||
{
|
||||
public CompressionCodec Codec => CompressionCodec.Deflate;
|
||||
|
||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
||||
{
|
||||
return CompressWithCodecStream(input, stream => new DeflateStream(stream, level, leaveOpen: true));
|
||||
}
|
||||
|
||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||
{
|
||||
return DecompressWithCodecStream(input, stream => new DeflateStream(stream, CompressionMode.Decompress, leaveOpen: true), expectedLength, maxDecompressedSizeBytes);
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] CompressWithCodecStream(ReadOnlySpan<byte> input, Func<Stream, Stream> streamFactory)
|
||||
{
|
||||
using var output = new MemoryStream(capacity: input.Length);
|
||||
using (var codecStream = streamFactory(output))
|
||||
{
|
||||
codecStream.Write(input);
|
||||
codecStream.Flush();
|
||||
}
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
private static byte[] DecompressWithCodecStream(
|
||||
ReadOnlySpan<byte> input,
|
||||
Func<Stream, Stream> streamFactory,
|
||||
int expectedLength,
|
||||
int maxDecompressedSizeBytes)
|
||||
{
|
||||
if (maxDecompressedSizeBytes <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(maxDecompressedSizeBytes));
|
||||
|
||||
using var compressed = new MemoryStream(input.ToArray(), writable: false);
|
||||
using var codecStream = streamFactory(compressed);
|
||||
using var output = expectedLength > 0
|
||||
? new MemoryStream(capacity: expectedLength)
|
||||
: new MemoryStream();
|
||||
|
||||
var buffer = ArrayPool<byte>.Shared.Rent(8192);
|
||||
try
|
||||
{
|
||||
int totalWritten = 0;
|
||||
while (true)
|
||||
{
|
||||
var bytesRead = codecStream.Read(buffer, 0, buffer.Length);
|
||||
if (bytesRead <= 0)
|
||||
break;
|
||||
|
||||
totalWritten += bytesRead;
|
||||
if (totalWritten > maxDecompressedSizeBytes)
|
||||
throw new InvalidDataException($"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
||||
|
||||
output.Write(buffer, 0, bytesRead);
|
||||
}
|
||||
|
||||
if (expectedLength >= 0 && totalWritten != expectedLength)
|
||||
throw new InvalidDataException($"Expected decompressed length {expectedLength}, actual {totalWritten}.");
|
||||
|
||||
return output.ToArray();
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
16
src/CBDD.Core/Compression/CompressionStats.cs
Normal file
16
src/CBDD.Core/Compression/CompressionStats.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Snapshot of aggregated compression and decompression telemetry.
|
||||
/// </summary>
|
||||
public readonly struct CompressionStats
|
||||
{
|
||||
public long CompressedDocumentCount { get; init; }
|
||||
public long BytesBeforeCompression { get; init; }
|
||||
public long BytesAfterCompression { get; init; }
|
||||
public long CompressionCpuTicks { get; init; }
|
||||
public long DecompressionCpuTicks { get; init; }
|
||||
public long CompressionFailureCount { get; init; }
|
||||
public long ChecksumFailureCount { get; init; }
|
||||
public long SafetyLimitRejectionCount { get; init; }
|
||||
}
|
||||
88
src/CBDD.Core/Compression/CompressionTelemetry.cs
Normal file
88
src/CBDD.Core/Compression/CompressionTelemetry.cs
Normal file
@@ -0,0 +1,88 @@
|
||||
using System.Threading;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Thread-safe counters for compression/decompression lifecycle events.
|
||||
/// </summary>
|
||||
public sealed class CompressionTelemetry
|
||||
{
|
||||
private long _compressionAttempts;
|
||||
private long _compressionSuccesses;
|
||||
private long _compressionFailures;
|
||||
private long _compressionSkippedTooSmall;
|
||||
private long _compressionSkippedInsufficientSavings;
|
||||
private long _decompressionAttempts;
|
||||
private long _decompressionSuccesses;
|
||||
private long _decompressionFailures;
|
||||
private long _compressionInputBytes;
|
||||
private long _compressionOutputBytes;
|
||||
private long _decompressionOutputBytes;
|
||||
private long _compressedDocumentCount;
|
||||
private long _compressionCpuTicks;
|
||||
private long _decompressionCpuTicks;
|
||||
private long _checksumFailureCount;
|
||||
private long _safetyLimitRejectionCount;
|
||||
|
||||
public long CompressionAttempts => Interlocked.Read(ref _compressionAttempts);
|
||||
public long CompressionSuccesses => Interlocked.Read(ref _compressionSuccesses);
|
||||
public long CompressionFailures => Interlocked.Read(ref _compressionFailures);
|
||||
public long CompressionSkippedTooSmall => Interlocked.Read(ref _compressionSkippedTooSmall);
|
||||
public long CompressionSkippedInsufficientSavings => Interlocked.Read(ref _compressionSkippedInsufficientSavings);
|
||||
public long DecompressionAttempts => Interlocked.Read(ref _decompressionAttempts);
|
||||
public long DecompressionSuccesses => Interlocked.Read(ref _decompressionSuccesses);
|
||||
public long DecompressionFailures => Interlocked.Read(ref _decompressionFailures);
|
||||
public long CompressionInputBytes => Interlocked.Read(ref _compressionInputBytes);
|
||||
public long CompressionOutputBytes => Interlocked.Read(ref _compressionOutputBytes);
|
||||
public long DecompressionOutputBytes => Interlocked.Read(ref _decompressionOutputBytes);
|
||||
public long CompressedDocumentCount => Interlocked.Read(ref _compressedDocumentCount);
|
||||
public long CompressionCpuTicks => Interlocked.Read(ref _compressionCpuTicks);
|
||||
public long DecompressionCpuTicks => Interlocked.Read(ref _decompressionCpuTicks);
|
||||
public long ChecksumFailureCount => Interlocked.Read(ref _checksumFailureCount);
|
||||
public long SafetyLimitRejectionCount => Interlocked.Read(ref _safetyLimitRejectionCount);
|
||||
|
||||
public void RecordCompressionAttempt(int inputBytes)
|
||||
{
|
||||
Interlocked.Increment(ref _compressionAttempts);
|
||||
Interlocked.Add(ref _compressionInputBytes, inputBytes);
|
||||
}
|
||||
|
||||
public void RecordCompressionSuccess(int outputBytes)
|
||||
{
|
||||
Interlocked.Increment(ref _compressionSuccesses);
|
||||
Interlocked.Increment(ref _compressedDocumentCount);
|
||||
Interlocked.Add(ref _compressionOutputBytes, outputBytes);
|
||||
}
|
||||
|
||||
public void RecordCompressionFailure() => Interlocked.Increment(ref _compressionFailures);
|
||||
public void RecordCompressionSkippedTooSmall() => Interlocked.Increment(ref _compressionSkippedTooSmall);
|
||||
public void RecordCompressionSkippedInsufficientSavings() => Interlocked.Increment(ref _compressionSkippedInsufficientSavings);
|
||||
public void RecordDecompressionAttempt() => Interlocked.Increment(ref _decompressionAttempts);
|
||||
public void RecordCompressionCpuTicks(long ticks) => Interlocked.Add(ref _compressionCpuTicks, ticks);
|
||||
public void RecordDecompressionCpuTicks(long ticks) => Interlocked.Add(ref _decompressionCpuTicks, ticks);
|
||||
public void RecordChecksumFailure() => Interlocked.Increment(ref _checksumFailureCount);
|
||||
public void RecordSafetyLimitRejection() => Interlocked.Increment(ref _safetyLimitRejectionCount);
|
||||
|
||||
public void RecordDecompressionSuccess(int outputBytes)
|
||||
{
|
||||
Interlocked.Increment(ref _decompressionSuccesses);
|
||||
Interlocked.Add(ref _decompressionOutputBytes, outputBytes);
|
||||
}
|
||||
|
||||
public void RecordDecompressionFailure() => Interlocked.Increment(ref _decompressionFailures);
|
||||
|
||||
public CompressionStats GetSnapshot()
|
||||
{
|
||||
return new CompressionStats
|
||||
{
|
||||
CompressedDocumentCount = CompressedDocumentCount,
|
||||
BytesBeforeCompression = CompressionInputBytes,
|
||||
BytesAfterCompression = CompressionOutputBytes,
|
||||
CompressionCpuTicks = CompressionCpuTicks,
|
||||
DecompressionCpuTicks = DecompressionCpuTicks,
|
||||
CompressionFailureCount = CompressionFailures,
|
||||
ChecksumFailureCount = ChecksumFailureCount,
|
||||
SafetyLimitRejectionCount = SafetyLimitRejectionCount
|
||||
};
|
||||
}
|
||||
}
|
||||
24
src/CBDD.Core/Compression/ICompressionCodec.cs
Normal file
24
src/CBDD.Core/Compression/ICompressionCodec.cs
Normal file
@@ -0,0 +1,24 @@
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
/// <summary>
|
||||
/// Codec abstraction for payload compression and decompression.
|
||||
/// </summary>
|
||||
public interface ICompressionCodec
|
||||
{
|
||||
/// <summary>
|
||||
/// Codec identifier.
|
||||
/// </summary>
|
||||
CompressionCodec Codec { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Compresses input bytes.
|
||||
/// </summary>
|
||||
byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level);
|
||||
|
||||
/// <summary>
|
||||
/// Decompresses payload bytes with output bounds validation.
|
||||
/// </summary>
|
||||
byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes);
|
||||
}
|
||||
450
src/CBDD.Core/DocumentDbContext.cs
Executable file
450
src/CBDD.Core/DocumentDbContext.cs
Executable file
@@ -0,0 +1,450 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using ZB.MOM.WW.CBDD.Core.Metadata;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using System.Threading;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for database contexts.
|
||||
/// Inherit and add DocumentCollection{T} properties for your entities.
|
||||
/// Use partial class for Source Generator integration.
|
||||
/// </summary>
|
||||
public abstract partial class DocumentDbContext : IDisposable, ITransactionHolder
|
||||
{
|
||||
private readonly IStorageEngine _storage;
|
||||
internal readonly CDC.ChangeStreamDispatcher _cdc;
|
||||
protected bool _disposed;
|
||||
private readonly SemaphoreSlim _transactionLock = new SemaphoreSlim(1, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current active transaction, if any.
|
||||
/// </summary>
|
||||
public ITransaction? CurrentTransaction
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
return field != null && (field.State == TransactionState.Active) ? field : null;
|
||||
}
|
||||
private set;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new database context with default configuration
|
||||
/// </summary>
|
||||
/// <param name="databasePath">The database file path.</param>
|
||||
protected DocumentDbContext(string databasePath)
|
||||
: this(databasePath, PageFileConfig.Default, CompressionOptions.Default)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new database context with default storage configuration and custom compression settings.
|
||||
/// </summary>
|
||||
/// <param name="databasePath">The database file path.</param>
|
||||
/// <param name="compressionOptions">Compression behavior options.</param>
|
||||
protected DocumentDbContext(string databasePath, CompressionOptions compressionOptions)
|
||||
: this(databasePath, PageFileConfig.Default, compressionOptions)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new database context with custom configuration
|
||||
/// </summary>
|
||||
/// <param name="databasePath">The database file path.</param>
|
||||
/// <param name="config">The page file configuration.</param>
|
||||
protected DocumentDbContext(string databasePath, PageFileConfig config)
|
||||
: this(databasePath, config, CompressionOptions.Default)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new database context with custom storage and compression configuration.
|
||||
/// </summary>
|
||||
/// <param name="databasePath">The database file path.</param>
|
||||
/// <param name="config">The page file configuration.</param>
|
||||
/// <param name="compressionOptions">Compression behavior options.</param>
|
||||
/// <param name="maintenanceOptions">Maintenance scheduling options.</param>
|
||||
protected DocumentDbContext(
|
||||
string databasePath,
|
||||
PageFileConfig config,
|
||||
CompressionOptions? compressionOptions,
|
||||
MaintenanceOptions? maintenanceOptions = null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(databasePath))
|
||||
throw new ArgumentNullException(nameof(databasePath));
|
||||
|
||||
_storage = new StorageEngine(databasePath, config, compressionOptions, maintenanceOptions);
|
||||
_cdc = new CDC.ChangeStreamDispatcher();
|
||||
_storage.RegisterCdc(_cdc);
|
||||
|
||||
// Initialize model before collections
|
||||
var modelBuilder = new ModelBuilder();
|
||||
OnModelCreating(modelBuilder);
|
||||
_model = modelBuilder.GetEntityBuilders();
|
||||
InitializeCollections();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes document collections for the context.
|
||||
/// </summary>
|
||||
protected virtual void InitializeCollections()
|
||||
{
|
||||
// Derived classes can override to initialize collections
|
||||
}
|
||||
|
||||
private readonly IReadOnlyDictionary<Type, object> _model;
|
||||
private readonly List<IDocumentMapper> _registeredMappers = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the concrete storage engine for advanced scenarios in derived contexts.
|
||||
/// </summary>
|
||||
protected StorageEngine Engine => (StorageEngine)_storage;
|
||||
|
||||
/// <summary>
|
||||
/// Gets compression options bound to this context's storage engine.
|
||||
/// </summary>
|
||||
protected CompressionOptions CompressionOptions => _storage.CompressionOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the compression service for codec operations.
|
||||
/// </summary>
|
||||
protected CompressionService CompressionService => _storage.CompressionService;
|
||||
|
||||
/// <summary>
|
||||
/// Gets compression telemetry counters.
|
||||
/// </summary>
|
||||
protected CompressionTelemetry CompressionTelemetry => _storage.CompressionTelemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Override to configure the model using Fluent API.
|
||||
/// </summary>
|
||||
/// <param name="modelBuilder">The model builder instance.</param>
|
||||
protected virtual void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper to create a DocumentCollection instance with custom TId.
|
||||
/// Used by derived classes in InitializeCollections for typed primary keys.
|
||||
/// </summary>
|
||||
/// <typeparam name="TId">The document identifier type.</typeparam>
|
||||
/// <typeparam name="T">The document type.</typeparam>
|
||||
/// <param name="mapper">The mapper used for document serialization and key access.</param>
|
||||
/// <returns>The created document collection.</returns>
|
||||
protected DocumentCollection<TId, T> CreateCollection<TId, T>(IDocumentMapper<TId, T> mapper)
|
||||
where T : class
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
string? customName = null;
|
||||
EntityTypeBuilder<T>? builder = null;
|
||||
|
||||
if (_model.TryGetValue(typeof(T), out var builderObj))
|
||||
{
|
||||
builder = builderObj as EntityTypeBuilder<T>;
|
||||
customName = builder?.CollectionName;
|
||||
}
|
||||
|
||||
_registeredMappers.Add(mapper);
|
||||
var collection = new DocumentCollection<TId, T>(_storage, this, mapper, customName);
|
||||
|
||||
// Apply configurations from ModelBuilder
|
||||
if (builder != null)
|
||||
{
|
||||
foreach (var indexBuilder in builder.Indexes)
|
||||
{
|
||||
collection.ApplyIndexBuilder(indexBuilder);
|
||||
}
|
||||
}
|
||||
|
||||
_storage.RegisterMappers(_registeredMappers);
|
||||
|
||||
return collection;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the document collection for the specified entity type using an ObjectId as the key.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The type of entity to retrieve the document collection for. Must be a reference type.</typeparam>
|
||||
/// <returns>A DocumentCollection<ObjectId, T> instance for the specified entity type.</returns>
|
||||
public DocumentCollection<ObjectId, T> Set<T>() where T : class => Set<ObjectId, T>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets a collection for managing documents of type T, identified by keys of type TId.
|
||||
/// Override is generated automatically by the Source Generator for partial DbContext classes.
|
||||
/// </summary>
|
||||
/// <typeparam name="TId">The type of the unique identifier for documents in the collection.</typeparam>
|
||||
/// <typeparam name="T">The type of the document to be managed. Must be a reference type.</typeparam>
|
||||
/// <returns>A DocumentCollection<TId, T> instance for performing operations on documents of type T.</returns>
|
||||
public virtual DocumentCollection<TId, T> Set<TId, T>() where T : class
|
||||
=> throw new InvalidOperationException($"No collection registered for entity type '{typeof(T).Name}' with key type '{typeof(TId).Name}'.");
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by the context.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
|
||||
_disposed = true;
|
||||
|
||||
_storage?.Dispose();
|
||||
_cdc?.Dispose();
|
||||
_transactionLock?.Dispose();
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins a transaction or returns the current active transaction.
|
||||
/// </summary>
|
||||
/// <returns>The active transaction.</returns>
|
||||
public ITransaction BeginTransaction()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
_transactionLock.Wait();
|
||||
try
|
||||
{
|
||||
if (CurrentTransaction != null)
|
||||
return CurrentTransaction; // Return existing active transaction
|
||||
CurrentTransaction = _storage.BeginTransaction();
|
||||
return CurrentTransaction;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_transactionLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins a transaction asynchronously or returns the current active transaction.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The active transaction.</returns>
|
||||
public async Task<ITransaction> BeginTransactionAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
bool lockAcquired = false;
|
||||
try
|
||||
{
|
||||
await _transactionLock.WaitAsync(ct);
|
||||
lockAcquired = true;
|
||||
|
||||
if (CurrentTransaction != null)
|
||||
return CurrentTransaction; // Return existing active transaction
|
||||
CurrentTransaction = await _storage.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct);
|
||||
return CurrentTransaction;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockAcquired)
|
||||
_transactionLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current active transaction or starts a new one.
|
||||
/// </summary>
|
||||
/// <returns>The active transaction.</returns>
|
||||
public ITransaction GetCurrentTransactionOrStart()
|
||||
{
|
||||
return BeginTransaction();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current active transaction or starts a new one asynchronously.
|
||||
/// </summary>
|
||||
/// <returns>The active transaction.</returns>
|
||||
public async Task<ITransaction> GetCurrentTransactionOrStartAsync()
|
||||
{
|
||||
return await BeginTransactionAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the current transaction if one is active.
|
||||
/// </summary>
|
||||
public void SaveChanges()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
if (CurrentTransaction != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
CurrentTransaction.Commit();
|
||||
}
|
||||
finally
|
||||
{
|
||||
CurrentTransaction = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the current transaction asynchronously if one is active.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
public async Task SaveChangesAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
if (CurrentTransaction != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await CurrentTransaction.CommitAsync(ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CurrentTransaction = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||
/// </summary>
|
||||
public CompressionStats GetCompressionStats()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetCompressionStats();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
||||
/// </summary>
|
||||
public CompactionStats Compact(CompactionOptions? options = null)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.Compact(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
||||
/// </summary>
|
||||
public Task<CompactionStats> CompactAsync(CompactionOptions? options = null, CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.CompactAsync(options, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Alias for <see cref="Compact(CompactionOptions?)"/>.
|
||||
/// </summary>
|
||||
public CompactionStats Vacuum(CompactionOptions? options = null)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.Vacuum(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Async alias for <see cref="CompactAsync(CompactionOptions?, CancellationToken)"/>.
|
||||
/// </summary>
|
||||
public Task<CompactionStats> VacuumAsync(CompactionOptions? options = null, CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.VacuumAsync(options, ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets page usage grouped by page type.
|
||||
/// </summary>
|
||||
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetPageUsageByPageType();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection page usage diagnostics.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetPageUsageByCollection();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection compression ratio diagnostics.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetCompressionRatioByCollection();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets free-list summary diagnostics.
|
||||
/// </summary>
|
||||
public FreeListSummary GetFreeListSummary()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetFreeListSummary();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets page-level fragmentation diagnostics.
|
||||
/// </summary>
|
||||
public FragmentationMapReport GetFragmentationMap()
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.GetFragmentationMap();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs compression migration as dry-run estimation by default.
|
||||
/// </summary>
|
||||
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.MigrateCompression(options);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs compression migration asynchronously as dry-run estimation by default.
|
||||
/// </summary>
|
||||
public Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null, CancellationToken ct = default)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||
|
||||
return Engine.MigrateCompressionAsync(options, ct);
|
||||
}
|
||||
}
|
||||
295
src/CBDD.Core/Indexing/BTreeCursor.cs
Executable file
295
src/CBDD.Core/Indexing/BTreeCursor.cs
Executable file
@@ -0,0 +1,295 @@
|
||||
using System.Buffers;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using System.Collections.Generic;
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
internal sealed class BTreeCursor : IBTreeCursor
|
||||
{
|
||||
private readonly BTreeIndex _index;
|
||||
private readonly ulong _transactionId;
|
||||
private readonly IIndexStorage _storage;
|
||||
|
||||
// State
|
||||
private byte[] _pageBuffer;
|
||||
private uint _currentPageId;
|
||||
private int _currentEntryIndex;
|
||||
private BTreeNodeHeader _currentHeader;
|
||||
private List<IndexEntry> _currentEntries;
|
||||
private bool _isValid;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BTreeCursor"/> class.
|
||||
/// </summary>
|
||||
/// <param name="index">The index to traverse.</param>
|
||||
/// <param name="storage">The storage engine for page access.</param>
|
||||
/// <param name="transactionId">The transaction identifier used for reads.</param>
|
||||
public BTreeCursor(BTreeIndex index, IIndexStorage storage, ulong transactionId)
|
||||
{
|
||||
_index = index;
|
||||
_storage = storage;
|
||||
_transactionId = transactionId;
|
||||
_pageBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||
_currentEntries = new List<IndexEntry>();
|
||||
_isValid = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current index entry at the cursor position.
|
||||
/// </summary>
|
||||
public IndexEntry Current
|
||||
{
|
||||
get
|
||||
{
|
||||
if (!_isValid) throw new InvalidOperationException("Cursor is not valid.");
|
||||
return _currentEntries[_currentEntryIndex];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the first entry in the index.
|
||||
/// </summary>
|
||||
/// <returns><see langword="true"/> if an entry is available; otherwise, <see langword="false"/>.</returns>
|
||||
public bool MoveToFirst()
|
||||
{
|
||||
// Find left-most leaf
|
||||
var pageId = _index.RootPageId;
|
||||
while (true)
|
||||
{
|
||||
LoadPage(pageId);
|
||||
if (_currentHeader.IsLeaf) break;
|
||||
|
||||
// Go to first child (P0)
|
||||
// Internal node format: [Header] [P0] [Entry1] ...
|
||||
var dataOffset = 32 + 20;
|
||||
pageId = BitConverter.ToUInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
||||
}
|
||||
|
||||
return PositionAtStart();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the last entry in the index.
|
||||
/// </summary>
|
||||
/// <returns><see langword="true"/> if an entry is available; otherwise, <see langword="false"/>.</returns>
|
||||
public bool MoveToLast()
|
||||
{
|
||||
// Find right-most leaf
|
||||
var pageId = _index.RootPageId;
|
||||
while (true)
|
||||
{
|
||||
LoadPage(pageId);
|
||||
if (_currentHeader.IsLeaf) break;
|
||||
|
||||
// Go to last child (last pointer)
|
||||
// Iterate all entries to find last pointer
|
||||
// P0 is at 32+20 (4 bytes). Entry 0 starts at 32+20+4.
|
||||
|
||||
// Wait, we need the last pointer.
|
||||
// P0 is at offset.
|
||||
// Then EncryCount entries: Key + Pointer.
|
||||
// We want the last pointer.
|
||||
|
||||
// Re-read P0 just in case
|
||||
uint lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(32 + 20, 4));
|
||||
|
||||
var offset = 32 + 20 + 4;
|
||||
for (int i = 0; i < _currentHeader.EntryCount; i++)
|
||||
{
|
||||
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(offset, 4));
|
||||
offset += 4 + keyLen;
|
||||
lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(offset, 4));
|
||||
offset += 4;
|
||||
}
|
||||
pageId = lastPointer;
|
||||
}
|
||||
|
||||
return PositionAtEnd();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeks to the specified key or the next greater key.
|
||||
/// </summary>
|
||||
/// <param name="key">The key to seek.</param>
|
||||
/// <returns>
|
||||
/// <see langword="true"/> if an exact key match is found; otherwise, <see langword="false"/>.
|
||||
/// </returns>
|
||||
public bool Seek(IndexKey key)
|
||||
{
|
||||
// Use Index to find leaf
|
||||
var leafPageId = _index.FindLeafNode(key, _transactionId);
|
||||
LoadPage(leafPageId);
|
||||
ParseEntries();
|
||||
|
||||
// Binary search in entries
|
||||
var idx = _currentEntries.BinarySearch(new IndexEntry(key, default(DocumentLocation)));
|
||||
|
||||
if (idx >= 0)
|
||||
{
|
||||
// Found exact match
|
||||
_currentEntryIndex = idx;
|
||||
_isValid = true;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Not found, ~idx is the next larger value
|
||||
_currentEntryIndex = ~idx;
|
||||
|
||||
if (_currentEntryIndex < _currentEntries.Count)
|
||||
{
|
||||
_isValid = true;
|
||||
return false; // Positioned at next greater
|
||||
}
|
||||
else
|
||||
{
|
||||
// Key is larger than max in this page, move to next page
|
||||
if (_currentHeader.NextLeafPageId != 0)
|
||||
{
|
||||
LoadPage(_currentHeader.NextLeafPageId);
|
||||
ParseEntries();
|
||||
_currentEntryIndex = 0;
|
||||
if (_currentEntries.Count > 0)
|
||||
{
|
||||
_isValid = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// End of index
|
||||
_isValid = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the next entry.
|
||||
/// </summary>
|
||||
/// <returns><see langword="true"/> if the cursor moved to a valid entry; otherwise, <see langword="false"/>.</returns>
|
||||
public bool MoveNext()
|
||||
{
|
||||
if (!_isValid) return false;
|
||||
|
||||
_currentEntryIndex++;
|
||||
if (_currentEntryIndex < _currentEntries.Count)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Move to next page
|
||||
if (_currentHeader.NextLeafPageId != 0)
|
||||
{
|
||||
LoadPage(_currentHeader.NextLeafPageId);
|
||||
return PositionAtStart();
|
||||
}
|
||||
|
||||
_isValid = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the previous entry.
|
||||
/// </summary>
|
||||
/// <returns><see langword="true"/> if the cursor moved to a valid entry; otherwise, <see langword="false"/>.</returns>
|
||||
public bool MovePrev()
|
||||
{
|
||||
if (!_isValid) return false;
|
||||
|
||||
_currentEntryIndex--;
|
||||
if (_currentEntryIndex >= 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Move to prev page
|
||||
if (_currentHeader.PrevLeafPageId != 0)
|
||||
{
|
||||
LoadPage(_currentHeader.PrevLeafPageId);
|
||||
return PositionAtEnd();
|
||||
}
|
||||
|
||||
_isValid = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
private void LoadPage(uint pageId)
|
||||
{
|
||||
if (_currentPageId == pageId && _pageBuffer != null) return;
|
||||
|
||||
_index.ReadPage(pageId, _transactionId, _pageBuffer);
|
||||
_currentPageId = pageId;
|
||||
_currentHeader = BTreeNodeHeader.ReadFrom(_pageBuffer.AsSpan(32));
|
||||
}
|
||||
|
||||
private void ParseEntries()
|
||||
{
|
||||
// Helper to parse entries from current page buffer
|
||||
// (Similar to BTreeIndex.ReadLeafEntries)
|
||||
_currentEntries.Clear();
|
||||
var dataOffset = 32 + 20;
|
||||
|
||||
for (int i = 0; i < _currentHeader.EntryCount; i++)
|
||||
{
|
||||
// Read Key
|
||||
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
||||
var keyData = new byte[keyLen];
|
||||
_pageBuffer.AsSpan(dataOffset + 4, keyLen).CopyTo(keyData);
|
||||
var key = new IndexKey(keyData);
|
||||
dataOffset += 4 + keyLen;
|
||||
|
||||
// Read Location
|
||||
var location = DocumentLocation.ReadFrom(_pageBuffer.AsSpan(dataOffset, DocumentLocation.SerializedSize));
|
||||
dataOffset += DocumentLocation.SerializedSize;
|
||||
|
||||
_currentEntries.Add(new IndexEntry(key, location));
|
||||
}
|
||||
}
|
||||
|
||||
private bool PositionAtStart()
|
||||
{
|
||||
ParseEntries();
|
||||
if (_currentEntries.Count > 0)
|
||||
{
|
||||
_currentEntryIndex = 0;
|
||||
_isValid = true;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Empty page? Should not happen in helper logic unless root leaf is empty
|
||||
_isValid = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private bool PositionAtEnd()
|
||||
{
|
||||
ParseEntries();
|
||||
if (_currentEntries.Count > 0)
|
||||
{
|
||||
_currentEntryIndex = _currentEntries.Count - 1;
|
||||
_isValid = true;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
_isValid = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases cursor resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_pageBuffer != null)
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(_pageBuffer);
|
||||
_pageBuffer = null!;
|
||||
}
|
||||
}
|
||||
}
|
||||
1248
src/CBDD.Core/Indexing/BTreeIndex.cs
Executable file
1248
src/CBDD.Core/Indexing/BTreeIndex.cs
Executable file
File diff suppressed because it is too large
Load Diff
157
src/CBDD.Core/Indexing/BTreeStructures.cs
Executable file
157
src/CBDD.Core/Indexing/BTreeStructures.cs
Executable file
@@ -0,0 +1,157 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Represents an entry in an index mapping a key to a document location.
|
||||
/// Implemented as struct for memory efficiency.
|
||||
/// </summary>
|
||||
public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the index key.
|
||||
/// </summary>
|
||||
public IndexKey Key { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the document location for the key.
|
||||
/// </summary>
|
||||
public DocumentLocation Location { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexEntry"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <param name="location">The document location.</param>
|
||||
public IndexEntry(IndexKey key, DocumentLocation location)
|
||||
{
|
||||
Key = key;
|
||||
Location = location;
|
||||
}
|
||||
|
||||
// Backward compatibility: constructor that takes ObjectId (for migration)
|
||||
// Will be removed once all code is migrated
|
||||
/// <summary>
|
||||
/// Initializes a legacy instance of the <see cref="IndexEntry"/> struct for migration scenarios.
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <param name="documentId">The legacy document identifier.</param>
|
||||
[Obsolete("Use constructor with DocumentLocation instead")]
|
||||
public IndexEntry(IndexKey key, ObjectId documentId)
|
||||
{
|
||||
Key = key;
|
||||
// Create a temporary location (will be replaced by proper implementation)
|
||||
Location = new DocumentLocation(0, 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares this entry to another entry by key.
|
||||
/// </summary>
|
||||
/// <param name="other">The other index entry to compare.</param>
|
||||
/// <returns>
|
||||
/// A value less than zero if this instance is less than <paramref name="other"/>,
|
||||
/// zero if they are equal, or greater than zero if this instance is greater.
|
||||
/// </returns>
|
||||
public int CompareTo(IndexEntry other)
|
||||
{
|
||||
return Key.CompareTo(other.Key);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares this entry to another object.
|
||||
/// </summary>
|
||||
/// <param name="obj">The object to compare.</param>
|
||||
/// <returns>
|
||||
/// A value less than zero if this instance is less than <paramref name="obj"/>,
|
||||
/// zero if they are equal, or greater than zero if this instance is greater.
|
||||
/// </returns>
|
||||
/// <exception cref="ArgumentException">Thrown when <paramref name="obj"/> is not an <see cref="IndexEntry"/>.</exception>
|
||||
public int CompareTo(object? obj)
|
||||
{
|
||||
if (obj is IndexEntry other) return CompareTo(other);
|
||||
throw new ArgumentException("Object is not an IndexEntry");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// B+Tree node for index storage.
|
||||
/// Uses struct for node metadata to minimize allocations.
|
||||
/// </summary>
|
||||
public struct BTreeNodeHeader
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the page identifier.
|
||||
/// </summary>
|
||||
public uint PageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether this node is a leaf node.
|
||||
/// </summary>
|
||||
public bool IsLeaf { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of entries in the node.
|
||||
/// </summary>
|
||||
public ushort EntryCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the parent page identifier.
|
||||
/// </summary>
|
||||
public uint ParentPageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the next leaf page identifier.
|
||||
/// </summary>
|
||||
public uint NextLeafPageId { get; set; } // For leaf nodes only
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the previous leaf page identifier.
|
||||
/// </summary>
|
||||
public uint PrevLeafPageId { get; set; } // For leaf nodes only (added for reverse scan)
|
||||
|
||||
/// <summary>
|
||||
/// Writes the header to a byte span.
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span.</param>
|
||||
public void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < 20)
|
||||
throw new ArgumentException("Destination must be at least 20 bytes");
|
||||
|
||||
BitConverter.TryWriteBytes(destination[0..4], PageId);
|
||||
destination[4] = (byte)(IsLeaf ? 1 : 0);
|
||||
BitConverter.TryWriteBytes(destination[5..7], EntryCount);
|
||||
BitConverter.TryWriteBytes(destination[7..11], ParentPageId);
|
||||
BitConverter.TryWriteBytes(destination[11..15], NextLeafPageId);
|
||||
BitConverter.TryWriteBytes(destination[15..19], PrevLeafPageId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a node header from a byte span.
|
||||
/// </summary>
|
||||
/// <param name="source">The source span.</param>
|
||||
/// <returns>The parsed node header.</returns>
|
||||
public static BTreeNodeHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < 20)
|
||||
throw new ArgumentException("Source must be at least 16 bytes");
|
||||
|
||||
var header = new BTreeNodeHeader
|
||||
{
|
||||
PageId = BitConverter.ToUInt32(source[0..4]),
|
||||
IsLeaf = source[4] != 0,
|
||||
EntryCount = BitConverter.ToUInt16(source[5..7]),
|
||||
ParentPageId = BitConverter.ToUInt32(source[7..11]),
|
||||
NextLeafPageId = BitConverter.ToUInt32(source[11..15])
|
||||
};
|
||||
|
||||
if (source.Length >= 20)
|
||||
{
|
||||
header.PrevLeafPageId = BitConverter.ToUInt32(source[15..19]);
|
||||
}
|
||||
|
||||
return header;
|
||||
}
|
||||
}
|
||||
202
src/CBDD.Core/Indexing/CollectionIndexDefinition.cs
Executable file
202
src/CBDD.Core/Indexing/CollectionIndexDefinition.cs
Executable file
@@ -0,0 +1,202 @@
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// High-level metadata and configuration for a custom index on a document collection.
|
||||
/// Wraps low-level IndexOptions and provides strongly-typed expression-based key extraction.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Document type</typeparam>
|
||||
public sealed class CollectionIndexDefinition<T> where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique name for this index (auto-generated or user-specified)
|
||||
/// </summary>
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Property paths that make up this index key.
|
||||
/// Examples: ["Age"] for simple index, ["City", "Age"] for compound index
|
||||
/// </summary>
|
||||
public string[] PropertyPaths { get; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, enforces uniqueness constraint on the indexed values
|
||||
/// </summary>
|
||||
public bool IsUnique { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Type of index structure (from existing IndexType enum)
|
||||
/// </summary>
|
||||
public IndexType Type { get; }
|
||||
|
||||
/// <summary>Vector dimensions (only for Vector index)</summary>
|
||||
public int Dimensions { get; }
|
||||
|
||||
/// <summary>Distance metric (only for Vector index)</summary>
|
||||
public VectorMetric Metric { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Compiled function to extract the index key from a document.
|
||||
/// Compiled for maximum performance (10-100x faster than interpreting Expression).
|
||||
/// </summary>
|
||||
public Func<T, object> KeySelector { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Original expression for the key selector (for analysis and serialization)
|
||||
/// </summary>
|
||||
public Expression<Func<T, object>> KeySelectorExpression { get; }
|
||||
|
||||
/// <summary>
|
||||
/// If true, this is the primary key index (_id)
|
||||
/// </summary>
|
||||
public bool IsPrimary { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new index definition
|
||||
/// </summary>
|
||||
/// <param name="name">Index name</param>
|
||||
/// <param name="propertyPaths">Property paths for the index</param>
|
||||
/// <param name="keySelectorExpression">Expression to extract key from document</param>
|
||||
/// <param name="isUnique">Enforce uniqueness</param>
|
||||
/// <param name="type">Index structure type (BTree or Hash)</param>
|
||||
/// <param name="isPrimary">Is this the primary key index</param>
|
||||
/// <param name="dimensions">The vector dimensions for vector indexes.</param>
|
||||
/// <param name="metric">The distance metric for vector indexes.</param>
|
||||
public CollectionIndexDefinition(
|
||||
string name,
|
||||
string[] propertyPaths,
|
||||
Expression<Func<T, object>> keySelectorExpression,
|
||||
bool isUnique = false,
|
||||
IndexType type = IndexType.BTree,
|
||||
bool isPrimary = false,
|
||||
int dimensions = 0,
|
||||
VectorMetric metric = VectorMetric.Cosine)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
throw new ArgumentException("Index name cannot be empty", nameof(name));
|
||||
|
||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||
throw new ArgumentException("Property paths cannot be empty", nameof(propertyPaths));
|
||||
|
||||
Name = name;
|
||||
PropertyPaths = propertyPaths;
|
||||
KeySelectorExpression = keySelectorExpression ?? throw new ArgumentNullException(nameof(keySelectorExpression));
|
||||
KeySelector = keySelectorExpression.Compile(); // Compile for performance
|
||||
IsUnique = isUnique;
|
||||
Type = type;
|
||||
IsPrimary = isPrimary;
|
||||
Dimensions = dimensions;
|
||||
Metric = metric;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts this high-level definition to low-level IndexOptions for BTreeIndex
|
||||
/// </summary>
|
||||
public IndexOptions ToIndexOptions()
|
||||
{
|
||||
return new IndexOptions
|
||||
{
|
||||
Type = Type,
|
||||
Unique = IsUnique,
|
||||
Fields = PropertyPaths,
|
||||
Dimensions = Dimensions,
|
||||
Metric = Metric
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if this index can be used for a query on the specified property path
|
||||
/// </summary>
|
||||
/// <param name="propertyPath">The property path to validate.</param>
|
||||
public bool CanSupportQuery(string propertyPath)
|
||||
{
|
||||
// Simple index: exact match required
|
||||
if (PropertyPaths.Length == 1)
|
||||
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Compound index: can support if queried property is the first component
|
||||
// e.g., index on ["City", "Age"] can support query on "City" but not just "Age"
|
||||
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if this index can support queries on multiple properties (compound queries)
|
||||
/// </summary>
|
||||
/// <param name="propertyPaths">The ordered property paths to validate.</param>
|
||||
public bool CanSupportCompoundQuery(string[] propertyPaths)
|
||||
{
|
||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||
return false;
|
||||
|
||||
// Check if queried paths are a prefix of this index
|
||||
// e.g., index on ["City", "Age", "Name"] can support ["City"] or ["City", "Age"]
|
||||
if (propertyPaths.Length > PropertyPaths.Length)
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < propertyPaths.Length; i++)
|
||||
{
|
||||
if (!PropertyPaths[i].Equals(propertyPaths[i], StringComparison.OrdinalIgnoreCase))
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString()
|
||||
{
|
||||
var uniqueStr = IsUnique ? "Unique" : "Non-Unique";
|
||||
var paths = string.Join(", ", PropertyPaths);
|
||||
return $"{Name} ({uniqueStr} {Type} on [{paths}])";
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Information about an existing index (for querying index metadata)
|
||||
/// </summary>
|
||||
public sealed class CollectionIndexInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the index name.
|
||||
/// </summary>
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the indexed property paths.
|
||||
/// </summary>
|
||||
public string[] PropertyPaths { get; init; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the index is unique.
|
||||
/// </summary>
|
||||
public bool IsUnique { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the index type.
|
||||
/// </summary>
|
||||
public IndexType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether this index is the primary index.
|
||||
/// </summary>
|
||||
public bool IsPrimary { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the estimated number of indexed documents.
|
||||
/// </summary>
|
||||
public long EstimatedDocumentCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the estimated storage size, in bytes.
|
||||
/// </summary>
|
||||
public long EstimatedSizeBytes { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string ToString()
|
||||
{
|
||||
return $"{Name}: {string.Join(", ", PropertyPaths)} ({EstimatedDocumentCount} docs, {EstimatedSizeBytes:N0} bytes)";
|
||||
}
|
||||
}
|
||||
607
src/CBDD.Core/Indexing/CollectionIndexManager.cs
Executable file
607
src/CBDD.Core/Indexing/CollectionIndexManager.cs
Executable file
@@ -0,0 +1,607 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq.Expressions;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Manages a collection of secondary indexes on a document collection.
|
||||
/// Handles index creation, deletion, automatic selection, and maintenance.
|
||||
/// </summary>
|
||||
/// <typeparam name="TId">Primary key type</typeparam>
|
||||
/// <typeparam name="T">Document type</typeparam>
|
||||
public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
||||
{
|
||||
private readonly Dictionary<string, CollectionSecondaryIndex<TId, T>> _indexes;
|
||||
private readonly IStorageEngine _storage;
|
||||
private readonly IDocumentMapper<TId, T> _mapper;
|
||||
private readonly object _lock = new();
|
||||
private bool _disposed;
|
||||
private readonly string _collectionName;
|
||||
private CollectionMetadata _metadata;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="CollectionIndexManager{TId, T}"/> class.
|
||||
/// </summary>
|
||||
/// <param name="storage">The storage engine used to persist index data and metadata.</param>
|
||||
/// <param name="mapper">The document mapper for the collection type.</param>
|
||||
/// <param name="collectionName">The optional collection name override.</param>
|
||||
public CollectionIndexManager(StorageEngine storage, IDocumentMapper<TId, T> mapper, string? collectionName = null)
|
||||
: this((IStorageEngine)storage, mapper, collectionName)
|
||||
{
|
||||
}
|
||||
|
||||
internal CollectionIndexManager(IStorageEngine storage, IDocumentMapper<TId, T> mapper, string? collectionName = null)
|
||||
{
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
||||
_collectionName = collectionName ?? _mapper.CollectionName;
|
||||
_indexes = new Dictionary<string, CollectionSecondaryIndex<TId, T>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Load existing metadata via storage
|
||||
_metadata = _storage.GetCollectionMetadata(_collectionName) ?? new CollectionMetadata { Name = _collectionName };
|
||||
|
||||
// Initialize indexes from metadata
|
||||
foreach (var idxMeta in _metadata.Indexes)
|
||||
{
|
||||
var definition = RebuildDefinition(idxMeta.Name, idxMeta.PropertyPaths, idxMeta.IsUnique, idxMeta.Type, idxMeta.Dimensions, idxMeta.Metric);
|
||||
var index = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper, idxMeta.RootPageId);
|
||||
_indexes[idxMeta.Name] = index;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateMetadata()
|
||||
{
|
||||
_metadata.Indexes.Clear();
|
||||
foreach (var index in _indexes.Values)
|
||||
{
|
||||
var info = index.GetInfo();
|
||||
_metadata.Indexes.Add(new IndexMetadata
|
||||
{
|
||||
Name = info.Name,
|
||||
IsUnique = info.IsUnique,
|
||||
Type = info.Type,
|
||||
PropertyPaths = info.PropertyPaths,
|
||||
Dimensions = index.Definition.Dimensions,
|
||||
Metric = index.Definition.Metric,
|
||||
RootPageId = index.RootPageId
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new secondary index
|
||||
/// </summary>
|
||||
/// <param name="definition">Index definition</param>
|
||||
/// <returns>The created secondary index</returns>
|
||||
public CollectionSecondaryIndex<TId, T> CreateIndex(CollectionIndexDefinition<T> definition)
|
||||
{
|
||||
if (definition == null)
|
||||
throw new ArgumentNullException(nameof(definition));
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(CollectionIndexManager<TId, T>));
|
||||
|
||||
// Check if index with this name already exists
|
||||
if (_indexes.ContainsKey(definition.Name))
|
||||
throw new InvalidOperationException($"Index '{definition.Name}' already exists");
|
||||
|
||||
// Create secondary index
|
||||
var secondaryIndex = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper);
|
||||
_indexes[definition.Name] = secondaryIndex;
|
||||
|
||||
// Persist metadata
|
||||
UpdateMetadata();
|
||||
_storage.SaveCollectionMetadata(_metadata);
|
||||
|
||||
return secondaryIndex;
|
||||
}
|
||||
}
|
||||
|
||||
// ... methods ...
|
||||
|
||||
/// <summary>
|
||||
/// Creates a simple index on a single property
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">Key type</typeparam>
|
||||
/// <param name="keySelector">Expression to extract key from document</param>
|
||||
/// <param name="name">Optional index name (auto-generated if null)</param>
|
||||
/// <param name="unique">Enforce uniqueness constraint</param>
|
||||
/// <returns>The created secondary index</returns>
|
||||
public CollectionSecondaryIndex<TId, T> CreateIndex<TKey>(
|
||||
Expression<Func<T, TKey>> keySelector,
|
||||
string? name = null,
|
||||
bool unique = false)
|
||||
{
|
||||
if (keySelector == null)
|
||||
throw new ArgumentNullException(nameof(keySelector));
|
||||
|
||||
// Extract property paths from expression
|
||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||
|
||||
// Generate name if not provided
|
||||
name ??= GenerateIndexName(propertyPaths);
|
||||
|
||||
// Convert expression to object-returning expression (required for definition)
|
||||
var objectSelector = Expression.Lambda<Func<T, object>>(
|
||||
Expression.Convert(keySelector.Body, typeof(object)),
|
||||
keySelector.Parameters);
|
||||
|
||||
// Create definition
|
||||
var definition = new CollectionIndexDefinition<T>(
|
||||
name,
|
||||
propertyPaths,
|
||||
objectSelector,
|
||||
unique);
|
||||
|
||||
return CreateIndex(definition);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a vector index for a collection property.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The selected key type.</typeparam>
|
||||
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
||||
/// <param name="dimensions">Vector dimensionality.</param>
|
||||
/// <param name="metric">Distance metric used by the vector index.</param>
|
||||
/// <param name="name">Optional index name.</param>
|
||||
/// <returns>The created or existing index.</returns>
|
||||
public CollectionSecondaryIndex<TId, T> CreateVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector, int dimensions, VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
||||
{
|
||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_indexes.TryGetValue(indexName, out var existing))
|
||||
return existing;
|
||||
|
||||
var body = keySelector.Body;
|
||||
if (body.Type != typeof(object))
|
||||
{
|
||||
body = Expression.Convert(body, typeof(object));
|
||||
}
|
||||
|
||||
// Reuse the original parameter from keySelector to avoid invalid expression trees.
|
||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||
|
||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric);
|
||||
return CreateIndex(definition);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures that an index exists for the specified key selector.
|
||||
/// </summary>
|
||||
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
||||
/// <param name="name">Optional index name.</param>
|
||||
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
||||
/// <returns>The existing or newly created index.</returns>
|
||||
public CollectionSecondaryIndex<TId, T> EnsureIndex(
|
||||
Expression<Func<T, object>> keySelector,
|
||||
string? name = null,
|
||||
bool unique = false)
|
||||
{
|
||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||
name ??= GenerateIndexName(propertyPaths);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_indexes.TryGetValue(name, out var existing))
|
||||
return existing;
|
||||
|
||||
return CreateIndex(keySelector, name, unique);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures that an index exists for the specified untyped key selector.
|
||||
/// </summary>
|
||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||
/// <param name="name">Optional index name.</param>
|
||||
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
||||
/// <returns>The existing or newly created index.</returns>
|
||||
internal CollectionSecondaryIndex<TId, T> EnsureIndexUntyped(
|
||||
LambdaExpression keySelector,
|
||||
string? name = null,
|
||||
bool unique = false)
|
||||
{
|
||||
// Convert LambdaExpression to Expression<Func<T, object>> properly by sharing parameters
|
||||
var body = keySelector.Body;
|
||||
if (body.Type != typeof(object))
|
||||
{
|
||||
body = Expression.Convert(body, typeof(object));
|
||||
}
|
||||
|
||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||
|
||||
return EnsureIndex(lambda, name, unique);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a vector index from an untyped key selector.
|
||||
/// </summary>
|
||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||
/// <param name="dimensions">Vector dimensionality.</param>
|
||||
/// <param name="metric">Distance metric used by the vector index.</param>
|
||||
/// <param name="name">Optional index name.</param>
|
||||
/// <returns>The created or existing index.</returns>
|
||||
public CollectionSecondaryIndex<TId, T> CreateVectorIndexUntyped(
|
||||
LambdaExpression keySelector,
|
||||
int dimensions,
|
||||
VectorMetric metric = VectorMetric.Cosine,
|
||||
string? name = null)
|
||||
{
|
||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_indexes.TryGetValue(indexName, out var existing))
|
||||
return existing;
|
||||
|
||||
var body = keySelector.Body;
|
||||
if (body.Type != typeof(object))
|
||||
{
|
||||
body = Expression.Convert(body, typeof(object));
|
||||
}
|
||||
|
||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||
|
||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric);
|
||||
return CreateIndex(definition);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a spatial index from an untyped key selector.
|
||||
/// </summary>
|
||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||
/// <param name="name">Optional index name.</param>
|
||||
/// <returns>The created or existing index.</returns>
|
||||
public CollectionSecondaryIndex<TId, T> CreateSpatialIndexUntyped(
|
||||
LambdaExpression keySelector,
|
||||
string? name = null)
|
||||
{
|
||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_indexes.TryGetValue(indexName, out var existing))
|
||||
return existing;
|
||||
|
||||
var body = keySelector.Body;
|
||||
if (body.Type != typeof(object))
|
||||
{
|
||||
body = Expression.Convert(body, typeof(object));
|
||||
}
|
||||
|
||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||
|
||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Spatial);
|
||||
return CreateIndex(definition);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops an existing index by name
|
||||
/// </summary>
|
||||
/// <param name="name">Index name</param>
|
||||
/// <returns>True if index was found and dropped, false otherwise</returns>
|
||||
public bool DropIndex(string name)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
throw new ArgumentException("Index name cannot be empty", nameof(name));
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (_indexes.TryGetValue(name, out var index))
|
||||
{
|
||||
index.Dispose();
|
||||
_indexes.Remove(name);
|
||||
|
||||
// TODO: Free pages used by index in PageFile
|
||||
|
||||
SaveMetadata(); // Save metadata after dropping index
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets an index by name
|
||||
/// </summary>
|
||||
/// <param name="name">The index name.</param>
|
||||
public CollectionSecondaryIndex<TId, T>? GetIndex(string name)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
return _indexes.TryGetValue(name, out var index) ? index : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all indexes
|
||||
/// </summary>
|
||||
public IEnumerable<CollectionSecondaryIndex<TId, T>> GetAllIndexes()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
return _indexes.Values.ToList(); // Return copy to avoid lock issues
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets information about all indexes
|
||||
/// </summary>
|
||||
public IEnumerable<CollectionIndexInfo> GetIndexInfo()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
return _indexes.Values.Select(idx => idx.GetInfo()).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the best index to use for a query on the specified property.
|
||||
/// Returns null if no suitable index found (requires full scan).
|
||||
/// </summary>
|
||||
/// <param name="propertyPath">Property path being queried</param>
|
||||
/// <returns>Best index for the query, or null if none suitable</returns>
|
||||
public CollectionSecondaryIndex<TId, T>? FindBestIndex(string propertyPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(propertyPath))
|
||||
return null;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
// Find all indexes that can support this query
|
||||
var candidates = _indexes.Values
|
||||
.Where(idx => idx.Definition.CanSupportQuery(propertyPath))
|
||||
.ToList();
|
||||
|
||||
if (candidates.Count == 0)
|
||||
return null;
|
||||
|
||||
// Simple strategy: prefer unique indexes, then shortest property path
|
||||
return candidates
|
||||
.OrderByDescending(idx => idx.Definition.IsUnique)
|
||||
.ThenBy(idx => idx.Definition.PropertyPaths.Length)
|
||||
.First();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the best index for a compound query on multiple properties
|
||||
/// </summary>
|
||||
/// <param name="propertyPaths">The ordered list of queried property paths.</param>
|
||||
public CollectionSecondaryIndex<TId, T>? FindBestCompoundIndex(string[] propertyPaths)
|
||||
{
|
||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||
return null;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
var candidates = _indexes.Values
|
||||
.Where(idx => idx.Definition.CanSupportCompoundQuery(propertyPaths))
|
||||
.ToList();
|
||||
|
||||
if (candidates.Count == 0)
|
||||
return null;
|
||||
|
||||
// Prefer longest matching index (more selective)
|
||||
return candidates
|
||||
.OrderByDescending(idx => idx.Definition.PropertyPaths.Length)
|
||||
.ThenByDescending(idx => idx.Definition.IsUnique)
|
||||
.First();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a document into all indexes
|
||||
/// </summary>
|
||||
/// <param name="document">Document to insert</param>
|
||||
/// <param name="location">Physical location of the document</param>
|
||||
/// <param name="transaction">Transaction context</param>
|
||||
public void InsertIntoAll(T document, DocumentLocation location, ITransaction transaction)
|
||||
{
|
||||
if (document == null)
|
||||
throw new ArgumentNullException(nameof(document));
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var index in _indexes.Values)
|
||||
{
|
||||
index.Insert(document, location, transaction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates a document in all indexes
|
||||
/// </summary>
|
||||
/// <param name="oldDocument">Old version of document</param>
|
||||
/// <param name="newDocument">New version of document</param>
|
||||
/// <param name="oldLocation">Physical location of old document</param>
|
||||
/// <param name="newLocation">Physical location of new document</param>
|
||||
/// <param name="transaction">Transaction context</param>
|
||||
public void UpdateInAll(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
|
||||
{
|
||||
if (oldDocument == null)
|
||||
throw new ArgumentNullException(nameof(oldDocument));
|
||||
if (newDocument == null)
|
||||
throw new ArgumentNullException(nameof(newDocument));
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var index in _indexes.Values)
|
||||
{
|
||||
index.Update(oldDocument, newDocument, oldLocation, newLocation, transaction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a document from all indexes
|
||||
/// </summary>
|
||||
/// <param name="document">Document to delete</param>
|
||||
/// <param name="location">Physical location of the document</param>
|
||||
/// <param name="transaction">Transaction context</param>
|
||||
public void DeleteFromAll(T document, DocumentLocation location, ITransaction transaction)
|
||||
{
|
||||
if (document == null)
|
||||
throw new ArgumentNullException(nameof(document));
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var index in _indexes.Values)
|
||||
{
|
||||
index.Delete(document, location, transaction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates an index name from property paths
|
||||
/// </summary>
|
||||
private static string GenerateIndexName(string[] propertyPaths)
|
||||
{
|
||||
return $"idx_{string.Join("_", propertyPaths)}";
|
||||
}
|
||||
|
||||
private CollectionIndexDefinition<T> RebuildDefinition(string name, string[] paths, bool isUnique, IndexType type, int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
||||
{
|
||||
var param = Expression.Parameter(typeof(T), "u");
|
||||
Expression body;
|
||||
|
||||
if (paths.Length == 1)
|
||||
{
|
||||
body = Expression.PropertyOrField(param, paths[0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
body = Expression.NewArrayInit(typeof(object),
|
||||
paths.Select(p => Expression.Convert(Expression.PropertyOrField(param, p), typeof(object))));
|
||||
}
|
||||
|
||||
var objectBody = Expression.Convert(body, typeof(object));
|
||||
var lambda = Expression.Lambda<Func<T, object>>(objectBody, param);
|
||||
|
||||
return new CollectionIndexDefinition<T>(name, paths, lambda, isUnique, type, false, dimensions, metric);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the root page identifier for the primary index.
|
||||
/// </summary>
|
||||
public uint PrimaryRootPageId => _metadata.PrimaryRootPageId;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the root page identifier for the primary index.
|
||||
/// </summary>
|
||||
/// <param name="pageId">The root page identifier.</param>
|
||||
public void SetPrimaryRootPageId(uint pageId)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_metadata.PrimaryRootPageId != pageId)
|
||||
{
|
||||
_metadata.PrimaryRootPageId = pageId;
|
||||
_storage.SaveCollectionMetadata(_metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current collection metadata.
|
||||
/// </summary>
|
||||
/// <returns>The collection metadata.</returns>
|
||||
public CollectionMetadata GetMetadata() => _metadata;
|
||||
|
||||
private void SaveMetadata()
|
||||
{
|
||||
UpdateMetadata();
|
||||
_storage.SaveCollectionMetadata(_metadata);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by the index manager.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
|
||||
// No auto-save on dispose to avoid unnecessary I/O if no changes
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
foreach (var index in _indexes.Values)
|
||||
{
|
||||
try { index.Dispose(); } catch { /* Best effort */ }
|
||||
}
|
||||
|
||||
_indexes.Clear();
|
||||
_disposed = true;
|
||||
}
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper class to analyze LINQ expressions and extract property paths
|
||||
/// </summary>
|
||||
public static class ExpressionAnalyzer
|
||||
{
|
||||
/// <summary>
|
||||
/// Extracts property paths from a lambda expression.
|
||||
/// Supports simple property access (p => p.Age) and anonymous types (p => new { p.City, p.Age }).
|
||||
/// </summary>
|
||||
/// <param name="expression">The lambda expression to analyze.</param>
|
||||
public static string[] ExtractPropertyPaths(LambdaExpression expression)
|
||||
{
|
||||
if (expression.Body is MemberExpression memberExpr)
|
||||
{
|
||||
// Simple property: p => p.Age
|
||||
return new[] { memberExpr.Member.Name };
|
||||
}
|
||||
else if (expression.Body is NewExpression newExpr)
|
||||
{
|
||||
// Compound key via anonymous type: p => new { p.City, p.Age }
|
||||
return newExpr.Arguments
|
||||
.OfType<MemberExpression>()
|
||||
.Select(m => m.Member.Name)
|
||||
.ToArray();
|
||||
}
|
||||
else if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } unaryExpr)
|
||||
{
|
||||
// Handle Convert(Member) or Convert(New)
|
||||
if (unaryExpr.Operand is MemberExpression innerMember)
|
||||
{
|
||||
// Wrapped property: p => (object)p.Age
|
||||
return new[] { innerMember.Member.Name };
|
||||
}
|
||||
else if (unaryExpr.Operand is NewExpression innerNew)
|
||||
{
|
||||
// Wrapped anonymous type: p => (object)new { p.City, p.Age }
|
||||
return innerNew.Arguments
|
||||
.OfType<MemberExpression>()
|
||||
.Select(m => m.Member.Name)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
throw new ArgumentException(
|
||||
"Expression must be a property accessor (p => p.Property) or anonymous type (p => new { p.Prop1, p.Prop2 })",
|
||||
nameof(expression));
|
||||
}
|
||||
}
|
||||
444
src/CBDD.Core/Indexing/CollectionSecondaryIndex.cs
Executable file
444
src/CBDD.Core/Indexing/CollectionSecondaryIndex.cs
Executable file
@@ -0,0 +1,444 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a secondary (non-primary) index on a document collection.
|
||||
/// Provides a high-level, strongly-typed wrapper around the low-level BTreeIndex.
|
||||
/// Handles automatic key extraction from documents using compiled expressions.
|
||||
/// </summary>
|
||||
/// <typeparam name="TId">Primary key type</typeparam>
|
||||
/// <typeparam name="T">Document type</typeparam>
|
||||
public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : class
|
||||
{
|
||||
private readonly CollectionIndexDefinition<T> _definition;
|
||||
private readonly BTreeIndex? _btreeIndex;
|
||||
private readonly VectorSearchIndex? _vectorIndex;
|
||||
private readonly RTreeIndex? _spatialIndex;
|
||||
private readonly IDocumentMapper<TId, T> _mapper;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the index definition
|
||||
/// </summary>
|
||||
public CollectionIndexDefinition<T> Definition => _definition;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the underlying BTree index (for advanced scenarios)
|
||||
/// </summary>
|
||||
public BTreeIndex? BTreeIndex => _btreeIndex;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the root page identifier for the underlying index structure.
|
||||
/// </summary>
|
||||
public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="CollectionSecondaryIndex{TId, T}"/> class.
|
||||
/// </summary>
|
||||
/// <param name="definition">The index definition.</param>
|
||||
/// <param name="storage">The storage engine.</param>
|
||||
/// <param name="mapper">The document mapper.</param>
|
||||
/// <param name="rootPageId">The existing root page ID, or <c>0</c> to create a new one.</param>
|
||||
public CollectionSecondaryIndex(
|
||||
CollectionIndexDefinition<T> definition,
|
||||
StorageEngine storage,
|
||||
IDocumentMapper<TId, T> mapper,
|
||||
uint rootPageId = 0)
|
||||
: this(definition, (IStorageEngine)storage, mapper, rootPageId)
|
||||
{
|
||||
}
|
||||
|
||||
internal CollectionSecondaryIndex(
|
||||
CollectionIndexDefinition<T> definition,
|
||||
IIndexStorage storage,
|
||||
IDocumentMapper<TId, T> mapper,
|
||||
uint rootPageId = 0)
|
||||
{
|
||||
_definition = definition ?? throw new ArgumentNullException(nameof(definition));
|
||||
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
||||
|
||||
var indexOptions = definition.ToIndexOptions();
|
||||
|
||||
if (indexOptions.Type == IndexType.Vector)
|
||||
{
|
||||
_vectorIndex = new VectorSearchIndex(storage, indexOptions, rootPageId);
|
||||
_btreeIndex = null;
|
||||
_spatialIndex = null;
|
||||
}
|
||||
else if (indexOptions.Type == IndexType.Spatial)
|
||||
{
|
||||
_spatialIndex = new RTreeIndex(storage, indexOptions, rootPageId);
|
||||
_btreeIndex = null;
|
||||
_vectorIndex = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
_btreeIndex = new BTreeIndex(storage, indexOptions, rootPageId);
|
||||
_vectorIndex = null;
|
||||
_spatialIndex = null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a document into this index
|
||||
/// </summary>
|
||||
/// <param name="document">Document to index</param>
|
||||
/// <param name="location">Physical location of the document</param>
|
||||
/// <param name="transaction">Optional transaction</param>
|
||||
public void Insert(T document, DocumentLocation location, ITransaction transaction)
|
||||
{
|
||||
if (document == null)
|
||||
throw new ArgumentNullException(nameof(document));
|
||||
|
||||
// Extract key using compiled selector (fast!)
|
||||
var keyValue = _definition.KeySelector(document);
|
||||
if (keyValue == null)
|
||||
return; // Skip null keys
|
||||
|
||||
if (_vectorIndex != null)
|
||||
{
|
||||
// Vector Index Support
|
||||
if (keyValue is float[] singleVector)
|
||||
{
|
||||
_vectorIndex.Insert(singleVector, location, transaction);
|
||||
}
|
||||
else if (keyValue is IEnumerable<float[]> vectors)
|
||||
{
|
||||
foreach (var v in vectors)
|
||||
{
|
||||
_vectorIndex.Insert(v, location, transaction);
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (_spatialIndex != null)
|
||||
{
|
||||
// Geospatial Index Support
|
||||
if (keyValue is ValueTuple<double, double> t)
|
||||
{
|
||||
_spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
|
||||
}
|
||||
}
|
||||
else if (_btreeIndex != null)
|
||||
{
|
||||
// BTree Index logic
|
||||
var userKey = ConvertToIndexKey(keyValue);
|
||||
var documentId = _mapper.GetId(document);
|
||||
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
||||
_btreeIndex.Insert(compositeKey, location, transaction?.TransactionId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates a document in this index (delete old, insert new).
|
||||
/// Only updates if the indexed key has changed.
|
||||
/// </summary>
|
||||
/// <param name="oldDocument">Old version of document</param>
|
||||
/// <param name="newDocument">New version of document</param>
|
||||
/// <param name="oldLocation">Physical location of old document</param>
|
||||
/// <param name="newLocation">Physical location of new document</param>
|
||||
/// <param name="transaction">Optional transaction</param>
|
||||
public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
|
||||
{
|
||||
if (oldDocument == null)
|
||||
throw new ArgumentNullException(nameof(oldDocument));
|
||||
if (newDocument == null)
|
||||
throw new ArgumentNullException(nameof(newDocument));
|
||||
|
||||
// Extract keys from both versions
|
||||
var oldKey = _definition.KeySelector(oldDocument);
|
||||
var newKey = _definition.KeySelector(newDocument);
|
||||
|
||||
// If keys are the same, no index update needed (optimization)
|
||||
if (Equals(oldKey, newKey))
|
||||
return;
|
||||
|
||||
var documentId = _mapper.GetId(oldDocument);
|
||||
|
||||
// Delete old entry if it had a key
|
||||
if (oldKey != null)
|
||||
{
|
||||
var oldUserKey = ConvertToIndexKey(oldKey);
|
||||
var oldCompositeKey = CreateCompositeKey(oldUserKey, _mapper.ToIndexKey(documentId));
|
||||
_btreeIndex?.Delete(oldCompositeKey, oldLocation, transaction?.TransactionId);
|
||||
}
|
||||
|
||||
// Insert new entry if it has a key
|
||||
if (newKey != null)
|
||||
{
|
||||
var newUserKey = ConvertToIndexKey(newKey);
|
||||
var newCompositeKey = CreateCompositeKey(newUserKey, _mapper.ToIndexKey(documentId));
|
||||
_btreeIndex?.Insert(newCompositeKey, newLocation, transaction?.TransactionId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a document from this index
|
||||
/// </summary>
|
||||
/// <param name="document">Document to remove from index</param>
|
||||
/// <param name="location">Physical location of the document</param>
|
||||
/// <param name="transaction">Optional transaction</param>
|
||||
public void Delete(T document, DocumentLocation location, ITransaction transaction)
|
||||
{
|
||||
if (document == null)
|
||||
throw new ArgumentNullException(nameof(document));
|
||||
|
||||
// Extract key
|
||||
var keyValue = _definition.KeySelector(document);
|
||||
if (keyValue == null)
|
||||
return; // Nothing to delete
|
||||
|
||||
var userKey = ConvertToIndexKey(keyValue);
|
||||
var documentId = _mapper.GetId(document);
|
||||
|
||||
// Create composite key and delete
|
||||
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
||||
_btreeIndex?.Delete(compositeKey, location, transaction?.TransactionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Seeks a single document by exact key match (O(log n))
|
||||
/// </summary>
|
||||
/// <param name="key">Key value to seek</param>
|
||||
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
||||
/// <returns>Document location if found, null otherwise</returns>
|
||||
public DocumentLocation? Seek(object key, ITransaction? transaction = null)
|
||||
{
|
||||
if (key == null)
|
||||
return null;
|
||||
|
||||
if (_vectorIndex != null && key is float[] query)
|
||||
{
|
||||
return _vectorIndex.Search(query, 1, transaction: transaction).FirstOrDefault().Location;
|
||||
}
|
||||
|
||||
if (_btreeIndex != null)
|
||||
{
|
||||
var userKey = ConvertToIndexKey(key);
|
||||
var minComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: true);
|
||||
var maxComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: false);
|
||||
var firstEntry = _btreeIndex.Range(minComposite, maxComposite, IndexDirection.Forward, transaction?.TransactionId).FirstOrDefault();
|
||||
return firstEntry.Location.PageId == 0 ? null : (DocumentLocation?)firstEntry.Location;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a vector nearest-neighbor search.
|
||||
/// </summary>
|
||||
/// <param name="query">The query vector.</param>
|
||||
/// <param name="k">The number of results to return.</param>
|
||||
/// <param name="efSearch">The search breadth parameter.</param>
|
||||
/// <param name="transaction">Optional transaction.</param>
|
||||
/// <returns>The matching vector search results.</returns>
|
||||
public IEnumerable<VectorSearchResult> VectorSearch(float[] query, int k, int efSearch = 100, ITransaction? transaction = null)
|
||||
{
|
||||
if (_vectorIndex == null)
|
||||
throw new InvalidOperationException("This index is not a vector index.");
|
||||
|
||||
return _vectorIndex.Search(query, k, efSearch, transaction);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs geospatial distance search
|
||||
/// </summary>
|
||||
/// <param name="center">The center point.</param>
|
||||
/// <param name="radiusKm">The search radius in kilometers.</param>
|
||||
/// <param name="transaction">Optional transaction.</param>
|
||||
public IEnumerable<DocumentLocation> Near((double Latitude, double Longitude) center, double radiusKm, ITransaction? transaction = null)
|
||||
{
|
||||
if (_spatialIndex == null)
|
||||
throw new InvalidOperationException("This index is not a spatial index.");
|
||||
|
||||
var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
||||
foreach (var loc in _spatialIndex.Search(queryBox, transaction))
|
||||
{
|
||||
yield return loc;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs geospatial bounding box search
|
||||
/// </summary>
|
||||
/// <param name="min">The minimum latitude/longitude corner.</param>
|
||||
/// <param name="max">The maximum latitude/longitude corner.</param>
|
||||
/// <param name="transaction">Optional transaction.</param>
|
||||
public IEnumerable<DocumentLocation> Within((double Latitude, double Longitude) min, (double Latitude, double Longitude) max, ITransaction? transaction = null)
|
||||
{
|
||||
if (_spatialIndex == null)
|
||||
throw new InvalidOperationException("This index is not a spatial index.");
|
||||
|
||||
var area = new GeoBox(min.Latitude, min.Longitude, max.Latitude, max.Longitude);
|
||||
return _spatialIndex.Search(area, transaction);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scans a range of keys (O(log n + k) where k is result count)
|
||||
/// </summary>
|
||||
/// <param name="minKey">Minimum key (inclusive), null for unbounded</param>
|
||||
/// <param name="maxKey">Maximum key (inclusive), null for unbounded</param>
|
||||
/// <param name="direction">Scan direction.</param>
|
||||
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
||||
/// <returns>Enumerable of document locations in key order</returns>
|
||||
public IEnumerable<DocumentLocation> Range(object? minKey, object? maxKey, IndexDirection direction = IndexDirection.Forward, ITransaction? transaction = null)
|
||||
{
|
||||
if (_btreeIndex == null) yield break;
|
||||
|
||||
// Handle unbounded ranges
|
||||
IndexKey actualMinKey;
|
||||
IndexKey actualMaxKey;
|
||||
|
||||
if (minKey == null && maxKey == null)
|
||||
{
|
||||
// Full scan - use extreme values
|
||||
actualMinKey = new IndexKey(new byte[0]); // Empty = smallest
|
||||
actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray()); // Max bytes
|
||||
}
|
||||
else if (minKey == null)
|
||||
{
|
||||
actualMinKey = new IndexKey(new byte[0]);
|
||||
var userMaxKey = ConvertToIndexKey(maxKey!);
|
||||
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false); // Max boundary
|
||||
}
|
||||
else if (maxKey == null)
|
||||
{
|
||||
var userMinKey = ConvertToIndexKey(minKey);
|
||||
actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true); // Min boundary
|
||||
actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray());
|
||||
}
|
||||
else
|
||||
{
|
||||
// Both bounds specified
|
||||
var userMinKey = ConvertToIndexKey(minKey);
|
||||
var userMaxKey = ConvertToIndexKey(maxKey);
|
||||
|
||||
// Create composite boundaries:
|
||||
// Min: (userMinKey, ObjectId.Empty) - captures all docs with key >= userMinKey
|
||||
// Max: (userMaxKey, ObjectId.MaxValue) - captures all docs with key <= userMaxKey
|
||||
actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true);
|
||||
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false);
|
||||
}
|
||||
|
||||
// Use BTreeIndex.Range with WAL-aware reads and direction
|
||||
// Extract DocumentLocation from each entry
|
||||
foreach (var entry in _btreeIndex.Range(actualMinKey, actualMaxKey, direction, transaction?.TransactionId))
|
||||
{
|
||||
yield return entry.Location;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets statistics about this index
|
||||
/// </summary>
|
||||
public CollectionIndexInfo GetInfo()
|
||||
{
|
||||
return new CollectionIndexInfo
|
||||
{
|
||||
Name = _definition.Name,
|
||||
PropertyPaths = _definition.PropertyPaths,
|
||||
IsUnique = _definition.IsUnique,
|
||||
Type = _definition.Type,
|
||||
IsPrimary = _definition.IsPrimary,
|
||||
EstimatedDocumentCount = 0, // TODO: Track or calculate document count
|
||||
EstimatedSizeBytes = 0 // TODO: Calculate index size
|
||||
};
|
||||
}
|
||||
|
||||
#region Composite Key Support (SQLite-style for Duplicate Keys)
|
||||
|
||||
/// <summary>
|
||||
/// Creates a composite key by concatenating user key with document ID.
|
||||
/// This allows duplicate user keys while maintaining BTree uniqueness.
|
||||
/// Format: [UserKeyBytes] + [DocumentIdKey]
|
||||
/// </summary>
|
||||
private IndexKey CreateCompositeKey(IndexKey userKey, IndexKey documentIdKey)
|
||||
{
|
||||
// Allocate buffer: user key + document ID key length
|
||||
var compositeBytes = new byte[userKey.Data.Length + documentIdKey.Data.Length];
|
||||
|
||||
// Copy user key
|
||||
userKey.Data.CopyTo(compositeBytes.AsSpan(0, userKey.Data.Length));
|
||||
|
||||
// Append document ID key
|
||||
documentIdKey.Data.CopyTo(compositeBytes.AsSpan(userKey.Data.Length));
|
||||
|
||||
return new IndexKey(compositeBytes);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a composite key for range query boundary.
|
||||
/// Uses MIN or MAX ID representation to capture all documents with the user key.
|
||||
/// </summary>
|
||||
private IndexKey CreateCompositeKeyBoundary(IndexKey userKey, bool useMinObjectId)
|
||||
{
|
||||
// For range boundaries, we use an empty key for Min and a very large key for Max
|
||||
// to wrap around all possible IDs for this user key.
|
||||
IndexKey idBoundary = useMinObjectId
|
||||
? new IndexKey(Array.Empty<byte>())
|
||||
: new IndexKey(Enumerable.Repeat((byte)0xFF, 16).ToArray()); // Using 16 as a safe max for GUID/ObjectId
|
||||
|
||||
return CreateCompositeKey(userKey, idBoundary);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts the original user key from a composite key by removing the ObjectId suffix.
|
||||
/// Used when we need to return the original indexed value.
|
||||
/// </summary>
|
||||
private IndexKey ExtractUserKey(IndexKey compositeKey)
|
||||
{
|
||||
// Composite key = UserKey + ObjectId(12 bytes)
|
||||
var userKeyLength = compositeKey.Data.Length - 12;
|
||||
if (userKeyLength <= 0)
|
||||
return compositeKey; // Fallback for malformed keys
|
||||
|
||||
var userKeyBytes = compositeKey.Data.Slice(0, userKeyLength);
|
||||
return new IndexKey(userKeyBytes);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Converts a CLR value to an IndexKey for BTree storage.
|
||||
/// Supports all common .NET types.
|
||||
/// </summary>
|
||||
private IndexKey ConvertToIndexKey(object value)
|
||||
{
|
||||
return value switch
|
||||
{
|
||||
ObjectId objectId => new IndexKey(objectId),
|
||||
string str => new IndexKey(str),
|
||||
int intVal => new IndexKey(intVal),
|
||||
long longVal => new IndexKey(longVal),
|
||||
DateTime dateTime => new IndexKey(dateTime.Ticks),
|
||||
bool boolVal => new IndexKey(boolVal ? 1 : 0),
|
||||
byte[] byteArray => new IndexKey(byteArray),
|
||||
|
||||
// For compound keys or complex types, use ToString and serialize
|
||||
// TODO: Better compound key serialization
|
||||
_ => new IndexKey(value.ToString() ?? string.Empty)
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by this index wrapper.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
|
||||
// BTreeIndex doesn't currently implement IDisposable
|
||||
// Future: may need to flush buffers, close resources
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
30
src/CBDD.Core/Indexing/GeoSpatialExtensions.cs
Executable file
30
src/CBDD.Core/Indexing/GeoSpatialExtensions.cs
Executable file
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
public static class GeoSpatialExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Performs a geospatial proximity search (Near) on a coordinate tuple property.
|
||||
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
||||
/// </summary>
|
||||
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
||||
/// <param name="center">The center point (Latitude, Longitude) for the proximity search.</param>
|
||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||
/// <returns>True if the point is within the specified radius.</returns>
|
||||
public static bool Near(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) center, double radiusKm)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a geospatial bounding box search (Within) on a coordinate tuple property.
|
||||
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
||||
/// </summary>
|
||||
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
||||
/// <param name="min">The minimum (Latitude, Longitude) of the bounding box.</param>
|
||||
/// <param name="max">The maximum (Latitude, Longitude) of the bounding box.</param>
|
||||
/// <returns>True if the point is within the specified bounding box.</returns>
|
||||
public static bool Within(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) min, (double Latitude, double Longitude) max)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
123
src/CBDD.Core/Indexing/HashIndex.cs
Executable file
123
src/CBDD.Core/Indexing/HashIndex.cs
Executable file
@@ -0,0 +1,123 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Hash-based index for exact-match lookups.
|
||||
/// Uses simple bucket-based hashing with collision handling.
|
||||
/// </summary>
|
||||
public sealed class HashIndex
|
||||
{
|
||||
private readonly Dictionary<int, List<IndexEntry>> _buckets;
|
||||
private readonly IndexOptions _options;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="HashIndex"/> class.
|
||||
/// </summary>
|
||||
/// <param name="options">The index options.</param>
|
||||
public HashIndex(IndexOptions options)
|
||||
{
|
||||
_options = options;
|
||||
_buckets = new Dictionary<int, List<IndexEntry>>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a key-location pair into the hash index
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <param name="location">The document location.</param>
|
||||
public void Insert(IndexKey key, DocumentLocation location)
|
||||
{
|
||||
if (_options.Unique && TryFind(key, out _))
|
||||
throw new InvalidOperationException($"Duplicate key violation for unique index");
|
||||
|
||||
var hashCode = key.GetHashCode();
|
||||
|
||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||
{
|
||||
bucket = new List<IndexEntry>();
|
||||
_buckets[hashCode] = bucket;
|
||||
}
|
||||
|
||||
bucket.Add(new IndexEntry(key, location));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds a document location by exact key match
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <param name="location">When this method returns, contains the matched document location if found.</param>
|
||||
/// <returns><see langword="true"/> if a matching entry is found; otherwise, <see langword="false"/>.</returns>
|
||||
public bool TryFind(IndexKey key, out DocumentLocation location)
|
||||
{
|
||||
location = default;
|
||||
var hashCode = key.GetHashCode();
|
||||
|
||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||
return false;
|
||||
|
||||
foreach (var entry in bucket)
|
||||
{
|
||||
if (entry.Key == key)
|
||||
{
|
||||
location = entry.Location;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes an entry from the index
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <param name="location">The document location.</param>
|
||||
/// <returns><see langword="true"/> if an entry is removed; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Remove(IndexKey key, DocumentLocation location)
|
||||
{
|
||||
var hashCode = key.GetHashCode();
|
||||
|
||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < bucket.Count; i++)
|
||||
{
|
||||
if (bucket[i].Key == key &&
|
||||
bucket[i].Location.PageId == location.PageId &&
|
||||
bucket[i].Location.SlotIndex == location.SlotIndex)
|
||||
{
|
||||
bucket.RemoveAt(i);
|
||||
|
||||
if (bucket.Count == 0)
|
||||
_buckets.Remove(hashCode);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all entries matching the key
|
||||
/// </summary>
|
||||
/// <param name="key">The index key.</param>
|
||||
/// <returns>All matching index entries.</returns>
|
||||
public IEnumerable<IndexEntry> FindAll(IndexKey key)
|
||||
{
|
||||
var hashCode = key.GetHashCode();
|
||||
|
||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||
yield break;
|
||||
|
||||
foreach (var entry in bucket)
|
||||
{
|
||||
if (entry.Key == key)
|
||||
yield return entry;
|
||||
}
|
||||
}
|
||||
}
|
||||
50
src/CBDD.Core/Indexing/IBTreeCursor.cs
Executable file
50
src/CBDD.Core/Indexing/IBTreeCursor.cs
Executable file
@@ -0,0 +1,50 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using System;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a cursor for traversing a B+Tree index.
|
||||
/// Provides low-level primitives for building complex queries.
|
||||
/// </summary>
|
||||
public interface IBTreeCursor : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the current entry at the cursor position.
|
||||
/// Throws InvalidOperationException if cursor is invalid or uninitialized.
|
||||
/// </summary>
|
||||
IndexEntry Current { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the first entry in the index.
|
||||
/// </summary>
|
||||
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
||||
bool MoveToFirst();
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the last entry in the index.
|
||||
/// </summary>
|
||||
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
||||
bool MoveToLast();
|
||||
|
||||
/// <summary>
|
||||
/// Seeks to the specified key.
|
||||
/// If exact match found, positions there and returns true.
|
||||
/// If not found, positions at the next greater key and returns false.
|
||||
/// </summary>
|
||||
/// <param name="key">Key to seek</param>
|
||||
/// <returns>True if exact match found; false if positioned at next greater key.</returns>
|
||||
bool Seek(IndexKey key);
|
||||
|
||||
/// <summary>
|
||||
/// Advances the cursor to the next entry.
|
||||
/// </summary>
|
||||
/// <returns>True if successfully moved; false if end of index reached.</returns>
|
||||
bool MoveNext();
|
||||
|
||||
/// <summary>
|
||||
/// Moves the cursor to the previous entry.
|
||||
/// </summary>
|
||||
/// <returns>True if successfully moved; false if start of index reached.</returns>
|
||||
bool MovePrev();
|
||||
}
|
||||
7
src/CBDD.Core/Indexing/IndexDirection.cs
Executable file
7
src/CBDD.Core/Indexing/IndexDirection.cs
Executable file
@@ -0,0 +1,7 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
public enum IndexDirection
|
||||
{
|
||||
Forward,
|
||||
Backward
|
||||
}
|
||||
191
src/CBDD.Core/Indexing/IndexKey.cs
Executable file
191
src/CBDD.Core/Indexing/IndexKey.cs
Executable file
@@ -0,0 +1,191 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using System;
|
||||
using System.Linq;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a key in an index.
|
||||
/// Implemented as struct for efficient index operations.
|
||||
/// Note: Contains byte array so cannot be readonly struct.
|
||||
/// </summary>
|
||||
public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
||||
{
|
||||
private readonly byte[] _data;
|
||||
private readonly int _hashCode;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the minimum possible index key.
|
||||
/// </summary>
|
||||
public static IndexKey MinKey => new IndexKey(Array.Empty<byte>());
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum possible index key.
|
||||
/// </summary>
|
||||
public static IndexKey MaxKey => new IndexKey(Enumerable.Repeat((byte)0xFF, 32).ToArray());
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from raw key bytes.
|
||||
/// </summary>
|
||||
/// <param name="data">The key bytes.</param>
|
||||
public IndexKey(ReadOnlySpan<byte> data)
|
||||
{
|
||||
_data = data.ToArray();
|
||||
_hashCode = ComputeHashCode(data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from an object identifier.
|
||||
/// </summary>
|
||||
/// <param name="objectId">The object identifier value.</param>
|
||||
public IndexKey(ObjectId objectId)
|
||||
{
|
||||
_data = new byte[12];
|
||||
objectId.WriteTo(_data);
|
||||
_hashCode = ComputeHashCode(_data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a 32-bit integer.
|
||||
/// </summary>
|
||||
/// <param name="value">The integer value.</param>
|
||||
public IndexKey(int value)
|
||||
{
|
||||
_data = BitConverter.GetBytes(value);
|
||||
_hashCode = ComputeHashCode(_data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a 64-bit integer.
|
||||
/// </summary>
|
||||
/// <param name="value">The integer value.</param>
|
||||
public IndexKey(long value)
|
||||
{
|
||||
_data = BitConverter.GetBytes(value);
|
||||
_hashCode = ComputeHashCode(_data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a string.
|
||||
/// </summary>
|
||||
/// <param name="value">The string value.</param>
|
||||
public IndexKey(string value)
|
||||
{
|
||||
_data = System.Text.Encoding.UTF8.GetBytes(value);
|
||||
_hashCode = ComputeHashCode(_data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a GUID.
|
||||
/// </summary>
|
||||
/// <param name="value">The GUID value.</param>
|
||||
public IndexKey(Guid value)
|
||||
{
|
||||
_data = value.ToByteArray();
|
||||
_hashCode = ComputeHashCode(_data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the raw byte data for this key.
|
||||
/// </summary>
|
||||
public readonly ReadOnlySpan<byte> Data => _data;
|
||||
|
||||
/// <summary>
|
||||
/// Compares this key to another key.
|
||||
/// </summary>
|
||||
/// <param name="other">The key to compare with.</param>
|
||||
/// <returns>
|
||||
/// A value less than zero if this key is less than <paramref name="other"/>, zero if equal, or greater than zero if greater.
|
||||
/// </returns>
|
||||
public readonly int CompareTo(IndexKey other)
|
||||
{
|
||||
if (_data == null) return other._data == null ? 0 : -1;
|
||||
if (other._data == null) return 1;
|
||||
|
||||
var minLength = Math.Min(_data.Length, other._data.Length);
|
||||
|
||||
for (int i = 0; i < minLength; i++)
|
||||
{
|
||||
var cmp = _data[i].CompareTo(other._data[i]);
|
||||
if (cmp != 0)
|
||||
return cmp;
|
||||
}
|
||||
|
||||
return _data.Length.CompareTo(other._data.Length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this key equals another key.
|
||||
/// </summary>
|
||||
/// <param name="other">The key to compare with.</param>
|
||||
/// <returns><see langword="true"/> if the keys are equal; otherwise, <see langword="false"/>.</returns>
|
||||
public readonly bool Equals(IndexKey other)
|
||||
{
|
||||
if (_hashCode != other._hashCode)
|
||||
return false;
|
||||
|
||||
if (_data == null) return other._data == null;
|
||||
if (other._data == null) return false;
|
||||
|
||||
return _data.AsSpan().SequenceEqual(other._data);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override readonly bool Equals(object? obj) => obj is IndexKey other && Equals(other);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override readonly int GetHashCode() => _hashCode;
|
||||
|
||||
public static bool operator ==(IndexKey left, IndexKey right) => left.Equals(right);
|
||||
public static bool operator !=(IndexKey left, IndexKey right) => !left.Equals(right);
|
||||
public static bool operator <(IndexKey left, IndexKey right) => left.CompareTo(right) < 0;
|
||||
public static bool operator >(IndexKey left, IndexKey right) => left.CompareTo(right) > 0;
|
||||
public static bool operator <=(IndexKey left, IndexKey right) => left.CompareTo(right) <= 0;
|
||||
public static bool operator >=(IndexKey left, IndexKey right) => left.CompareTo(right) >= 0;
|
||||
|
||||
private static int ComputeHashCode(ReadOnlySpan<byte> data)
|
||||
{
|
||||
var hash = new HashCode();
|
||||
hash.AddBytes(data);
|
||||
return hash.ToHashCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates an <see cref="IndexKey"/> from a supported CLR value.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The CLR type of the value.</typeparam>
|
||||
/// <param name="value">The value to convert.</param>
|
||||
/// <returns>The created index key.</returns>
|
||||
public static IndexKey Create<T>(T value)
|
||||
{
|
||||
if (value == null) return default;
|
||||
|
||||
if (typeof(T) == typeof(ObjectId)) return new IndexKey((ObjectId)(object)value);
|
||||
if (typeof(T) == typeof(int)) return new IndexKey((int)(object)value);
|
||||
if (typeof(T) == typeof(long)) return new IndexKey((long)(object)value);
|
||||
if (typeof(T) == typeof(string)) return new IndexKey((string)(object)value);
|
||||
if (typeof(T) == typeof(Guid)) return new IndexKey((Guid)(object)value);
|
||||
if (typeof(T) == typeof(byte[])) return new IndexKey((byte[])(object)value);
|
||||
|
||||
throw new NotSupportedException($"Type {typeof(T).Name} is not supported as an IndexKey. Provide a custom mapping.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts this key to a CLR value of type <typeparamref name="T"/>.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The CLR type to read from this key.</typeparam>
|
||||
/// <returns>The converted value.</returns>
|
||||
public readonly T As<T>()
|
||||
{
|
||||
if (_data == null) return default!;
|
||||
|
||||
if (typeof(T) == typeof(ObjectId)) return (T)(object)new ObjectId(_data);
|
||||
if (typeof(T) == typeof(int)) return (T)(object)BitConverter.ToInt32(_data);
|
||||
if (typeof(T) == typeof(long)) return (T)(object)BitConverter.ToInt64(_data);
|
||||
if (typeof(T) == typeof(string)) return (T)(object)System.Text.Encoding.UTF8.GetString(_data);
|
||||
if (typeof(T) == typeof(Guid)) return (T)(object)new Guid(_data);
|
||||
if (typeof(T) == typeof(byte[])) return (T)(object)_data;
|
||||
|
||||
throw new NotSupportedException($"Type {typeof(T).Name} cannot be extracted from IndexKey. Provide a custom mapping.");
|
||||
}
|
||||
}
|
||||
148
src/CBDD.Core/Indexing/IndexOptions.cs
Executable file
148
src/CBDD.Core/Indexing/IndexOptions.cs
Executable file
@@ -0,0 +1,148 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Types of indices supported
|
||||
/// </summary>
|
||||
public enum IndexType : byte
|
||||
{
|
||||
/// <summary>B+Tree index for range queries and ordering</summary>
|
||||
BTree = 1,
|
||||
|
||||
/// <summary>Hash index for exact match lookups</summary>
|
||||
Hash = 2,
|
||||
|
||||
/// <summary>Unique index constraint</summary>
|
||||
Unique = 3,
|
||||
|
||||
/// <summary>Vector index (HNSW) for similarity search</summary>
|
||||
Vector = 4,
|
||||
|
||||
/// <summary>Geospatial index (R-Tree) for spatial queries</summary>
|
||||
Spatial = 5
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Distance metrics for vector search
|
||||
/// </summary>
|
||||
public enum VectorMetric : byte
|
||||
{
|
||||
/// <summary>Cosine Similarity (Standard for embeddings)</summary>
|
||||
Cosine = 1,
|
||||
|
||||
/// <summary>Euclidean Distance (L2)</summary>
|
||||
L2 = 2,
|
||||
|
||||
/// <summary>Dot Product</summary>
|
||||
DotProduct = 3
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Index options and configuration.
|
||||
/// Implemented as readonly struct for efficiency.
|
||||
/// </summary>
|
||||
public readonly struct IndexOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the configured index type.
|
||||
/// </summary>
|
||||
public IndexType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the index enforces uniqueness.
|
||||
/// </summary>
|
||||
public bool Unique { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the indexed field names.
|
||||
/// </summary>
|
||||
public string[] Fields { get; init; }
|
||||
|
||||
// Vector search options
|
||||
/// <summary>
|
||||
/// Gets the vector dimensionality for vector indexes.
|
||||
/// </summary>
|
||||
public int Dimensions { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the distance metric used for vector similarity.
|
||||
/// </summary>
|
||||
public VectorMetric Metric { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the minimum number of graph connections per node.
|
||||
/// </summary>
|
||||
public int M { get; init; } // Min number of connections per node
|
||||
|
||||
/// <summary>
|
||||
/// Gets the size of the dynamic candidate list during index construction.
|
||||
/// </summary>
|
||||
public int EfConstruction { get; init; } // Size of dynamic candidate list for construction
|
||||
|
||||
/// <summary>
|
||||
/// Creates non-unique B+Tree index options.
|
||||
/// </summary>
|
||||
/// <param name="fields">The indexed field names.</param>
|
||||
/// <returns>The configured index options.</returns>
|
||||
public static IndexOptions CreateBTree(params string[] fields) => new()
|
||||
{
|
||||
Type = IndexType.BTree,
|
||||
Unique = false,
|
||||
Fields = fields
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates unique B+Tree index options.
|
||||
/// </summary>
|
||||
/// <param name="fields">The indexed field names.</param>
|
||||
/// <returns>The configured index options.</returns>
|
||||
public static IndexOptions CreateUnique(params string[] fields) => new()
|
||||
{
|
||||
Type = IndexType.BTree,
|
||||
Unique = true,
|
||||
Fields = fields
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates hash index options.
|
||||
/// </summary>
|
||||
/// <param name="fields">The indexed field names.</param>
|
||||
/// <returns>The configured index options.</returns>
|
||||
public static IndexOptions CreateHash(params string[] fields) => new()
|
||||
{
|
||||
Type = IndexType.Hash,
|
||||
Unique = false,
|
||||
Fields = fields
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates vector index options.
|
||||
/// </summary>
|
||||
/// <param name="dimensions">The vector dimensionality.</param>
|
||||
/// <param name="metric">The similarity metric.</param>
|
||||
/// <param name="m">The minimum number of graph connections per node.</param>
|
||||
/// <param name="ef">The candidate list size used during index construction.</param>
|
||||
/// <param name="fields">The indexed field names.</param>
|
||||
/// <returns>The configured index options.</returns>
|
||||
public static IndexOptions CreateVector(int dimensions, VectorMetric metric = VectorMetric.Cosine, int m = 16, int ef = 200, params string[] fields) => new()
|
||||
{
|
||||
Type = IndexType.Vector,
|
||||
Unique = false,
|
||||
Fields = fields,
|
||||
Dimensions = dimensions,
|
||||
Metric = metric,
|
||||
M = m,
|
||||
EfConstruction = ef
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates spatial index options.
|
||||
/// </summary>
|
||||
/// <param name="fields">The indexed field names.</param>
|
||||
/// <returns>The configured index options.</returns>
|
||||
public static IndexOptions CreateSpatial(params string[] fields) => new()
|
||||
{
|
||||
Type = IndexType.Spatial,
|
||||
Unique = false,
|
||||
Fields = fields
|
||||
};
|
||||
}
|
||||
90
src/CBDD.Core/Indexing/Internal/GeoTypes.cs
Executable file
90
src/CBDD.Core/Indexing/Internal/GeoTypes.cs
Executable file
@@ -0,0 +1,90 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Basic spatial point (Latitude/Longitude)
|
||||
/// Internal primitive for R-Tree logic.
|
||||
/// </summary>
|
||||
internal record struct GeoPoint(double Latitude, double Longitude)
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets an empty point at coordinate origin.
|
||||
/// </summary>
|
||||
public static GeoPoint Empty => new(0, 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimum Bounding Box (MBR) for spatial indexing
|
||||
/// Internal primitive for R-Tree logic.
|
||||
/// </summary>
|
||||
internal record struct GeoBox(double MinLat, double MinLon, double MaxLat, double MaxLon)
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets an empty bounding box sentinel value.
|
||||
/// </summary>
|
||||
public static GeoBox Empty => new(double.MaxValue, double.MaxValue, double.MinValue, double.MinValue);
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this box contains the specified point.
|
||||
/// </summary>
|
||||
/// <param name="point">The point to test.</param>
|
||||
/// <returns><see langword="true"/> if the point is inside this box; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Contains(GeoPoint point)
|
||||
{
|
||||
return point.Latitude >= MinLat && point.Latitude <= MaxLat &&
|
||||
point.Longitude >= MinLon && point.Longitude <= MaxLon;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether this box intersects another box.
|
||||
/// </summary>
|
||||
/// <param name="other">The other box to test.</param>
|
||||
/// <returns><see langword="true"/> if the boxes intersect; otherwise, <see langword="false"/>.</returns>
|
||||
public bool Intersects(GeoBox other)
|
||||
{
|
||||
return !(other.MinLat > MaxLat || other.MaxLat < MinLat ||
|
||||
other.MinLon > MaxLon || other.MaxLon < MinLon);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a box that contains a single point.
|
||||
/// </summary>
|
||||
/// <param name="point">The point to convert.</param>
|
||||
/// <returns>A bounding box containing the specified point.</returns>
|
||||
public static GeoBox FromPoint(GeoPoint point)
|
||||
{
|
||||
return new GeoBox(point.Latitude, point.Longitude, point.Latitude, point.Longitude);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands this box to include the specified point.
|
||||
/// </summary>
|
||||
/// <param name="point">The point to include.</param>
|
||||
/// <returns>A new expanded bounding box.</returns>
|
||||
public GeoBox ExpandTo(GeoPoint point)
|
||||
{
|
||||
return new GeoBox(
|
||||
Math.Min(MinLat, point.Latitude),
|
||||
Math.Min(MinLon, point.Longitude),
|
||||
Math.Max(MaxLat, point.Latitude),
|
||||
Math.Max(MaxLon, point.Longitude));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands this box to include the specified box.
|
||||
/// </summary>
|
||||
/// <param name="other">The box to include.</param>
|
||||
/// <returns>A new expanded bounding box.</returns>
|
||||
public GeoBox ExpandTo(GeoBox other)
|
||||
{
|
||||
return new GeoBox(
|
||||
Math.Min(MinLat, other.MinLat),
|
||||
Math.Min(MinLon, other.MinLon),
|
||||
Math.Max(MaxLat, other.MaxLat),
|
||||
Math.Max(MaxLon, other.MaxLon));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the area of this bounding box.
|
||||
/// </summary>
|
||||
public double Area => Math.Max(0, MaxLat - MinLat) * Math.Max(0, MaxLon - MinLon);
|
||||
}
|
||||
27
src/CBDD.Core/Indexing/InternalEntry.cs
Executable file
27
src/CBDD.Core/Indexing/InternalEntry.cs
Executable file
@@ -0,0 +1,27 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
public struct InternalEntry
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the separator key.
|
||||
/// </summary>
|
||||
public IndexKey Key { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the child page identifier.
|
||||
/// </summary>
|
||||
public uint PageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="InternalEntry"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="key">The separator key.</param>
|
||||
/// <param name="pageId">The child page identifier.</param>
|
||||
public InternalEntry(IndexKey key, uint pageId)
|
||||
{
|
||||
Key = key;
|
||||
PageId = pageId;
|
||||
}
|
||||
}
|
||||
410
src/CBDD.Core/Indexing/RTreeIndex.cs
Executable file
410
src/CBDD.Core/Indexing/RTreeIndex.cs
Executable file
@@ -0,0 +1,410 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||
using System.Buffers;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// R-Tree Index implementation for Geospatial Indexing.
|
||||
/// Uses Quadratic Split algorithm for simplicity and efficiency in paged storage.
|
||||
/// </summary>
|
||||
internal class RTreeIndex : IDisposable
|
||||
{
|
||||
private readonly IIndexStorage _storage;
|
||||
private readonly IndexOptions _options;
|
||||
private uint _rootPageId;
|
||||
private readonly object _lock = new();
|
||||
private readonly int _pageSize;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RTreeIndex"/> class.
|
||||
/// </summary>
|
||||
/// <param name="storage">The storage engine used for page operations.</param>
|
||||
/// <param name="options">The index options.</param>
|
||||
/// <param name="rootPageId">The root page identifier, or <c>0</c> to create a new root.</param>
|
||||
public RTreeIndex(IIndexStorage storage, IndexOptions options, uint rootPageId)
|
||||
{
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_options = options;
|
||||
_rootPageId = rootPageId;
|
||||
_pageSize = _storage.PageSize;
|
||||
|
||||
if (_rootPageId == 0)
|
||||
{
|
||||
InitializeNewIndex();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current root page identifier.
|
||||
/// </summary>
|
||||
public uint RootPageId => _rootPageId;
|
||||
|
||||
private void InitializeNewIndex()
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_rootPageId = _storage.AllocatePage();
|
||||
SpatialPage.Initialize(buffer, _rootPageId, true, 0);
|
||||
_storage.WritePageImmediate(_rootPageId, buffer);
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Searches for document locations whose minimum bounding rectangles intersect the specified area.
|
||||
/// </summary>
|
||||
/// <param name="area">The area to search.</param>
|
||||
/// <param name="transaction">The optional transaction context.</param>
|
||||
/// <returns>A sequence of matching document locations.</returns>
|
||||
public IEnumerable<DocumentLocation> Search(GeoBox area, ITransaction? transaction = null)
|
||||
{
|
||||
if (_rootPageId == 0) yield break;
|
||||
|
||||
var stack = new Stack<uint>();
|
||||
stack.Push(_rootPageId);
|
||||
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
while (stack.Count > 0)
|
||||
{
|
||||
uint pageId = stack.Pop();
|
||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||
|
||||
bool isLeaf = SpatialPage.GetIsLeaf(buffer);
|
||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
SpatialPage.ReadEntry(buffer, i, out var mbr, out var pointer);
|
||||
|
||||
if (area.Intersects(mbr))
|
||||
{
|
||||
if (isLeaf)
|
||||
{
|
||||
yield return pointer;
|
||||
}
|
||||
else
|
||||
{
|
||||
stack.Push(pointer.PageId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a bounding rectangle and document location into the index.
|
||||
/// </summary>
|
||||
/// <param name="mbr">The minimum bounding rectangle to index.</param>
|
||||
/// <param name="loc">The document location associated with the rectangle.</param>
|
||||
/// <param name="transaction">The optional transaction context.</param>
|
||||
public void Insert(GeoBox mbr, DocumentLocation loc, ITransaction? transaction = null)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
var leafPageId = ChooseLeaf(_rootPageId, mbr, transaction);
|
||||
InsertIntoNode(leafPageId, mbr, loc, transaction);
|
||||
}
|
||||
}
|
||||
|
||||
private uint ChooseLeaf(uint rootId, GeoBox mbr, ITransaction? transaction)
|
||||
{
|
||||
uint currentId = rootId;
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
_storage.ReadPage(currentId, transaction?.TransactionId, buffer);
|
||||
if (SpatialPage.GetIsLeaf(buffer)) return currentId;
|
||||
|
||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||
uint bestChild = 0;
|
||||
double minEnlargement = double.MaxValue;
|
||||
double minArea = double.MaxValue;
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
SpatialPage.ReadEntry(buffer, i, out var childMbr, out var pointer);
|
||||
|
||||
var expanded = childMbr.ExpandTo(mbr);
|
||||
double enlargement = expanded.Area - childMbr.Area;
|
||||
|
||||
if (enlargement < minEnlargement)
|
||||
{
|
||||
minEnlargement = enlargement;
|
||||
minArea = childMbr.Area;
|
||||
bestChild = pointer.PageId;
|
||||
}
|
||||
else if (enlargement == minEnlargement)
|
||||
{
|
||||
if (childMbr.Area < minArea)
|
||||
{
|
||||
minArea = childMbr.Area;
|
||||
bestChild = pointer.PageId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentId = bestChild;
|
||||
}
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private void InsertIntoNode(uint pageId, GeoBox mbr, DocumentLocation pointer, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||
int maxEntries = SpatialPage.GetMaxEntries(_pageSize);
|
||||
|
||||
if (count < maxEntries)
|
||||
{
|
||||
SpatialPage.WriteEntry(buffer, count, mbr, pointer);
|
||||
SpatialPage.SetEntryCount(buffer, (ushort)(count + 1));
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||
else
|
||||
_storage.WritePageImmediate(pageId, buffer);
|
||||
|
||||
// Propagate MBR update upwards
|
||||
UpdateMBRUpwards(pageId, transaction);
|
||||
}
|
||||
else
|
||||
{
|
||||
SplitNode(pageId, mbr, pointer, transaction);
|
||||
}
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private void UpdateMBRUpwards(uint pageId, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
var parentBuffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
uint currentId = pageId;
|
||||
while (currentId != _rootPageId)
|
||||
{
|
||||
_storage.ReadPage(currentId, transaction?.TransactionId, buffer);
|
||||
var currentMbr = SpatialPage.CalculateMBR(buffer);
|
||||
uint parentId = SpatialPage.GetParentPageId(buffer);
|
||||
|
||||
if (parentId == 0) break;
|
||||
|
||||
_storage.ReadPage(parentId, transaction?.TransactionId, parentBuffer);
|
||||
ushort count = SpatialPage.GetEntryCount(parentBuffer);
|
||||
bool changed = false;
|
||||
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
SpatialPage.ReadEntry(parentBuffer, i, out var mbr, out var pointer);
|
||||
if (pointer.PageId == currentId)
|
||||
{
|
||||
if (mbr != currentMbr)
|
||||
{
|
||||
SpatialPage.WriteEntry(parentBuffer, i, currentMbr, pointer);
|
||||
changed = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!changed) break;
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(parentId, transaction.TransactionId, parentBuffer);
|
||||
else
|
||||
_storage.WritePageImmediate(parentId, parentBuffer);
|
||||
|
||||
currentId = parentId;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReturnPageBuffer(buffer);
|
||||
ReturnPageBuffer(parentBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
private void SplitNode(uint pageId, GeoBox newMbr, DocumentLocation newPointer, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
var newBuffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||
bool isLeaf = SpatialPage.GetIsLeaf(buffer);
|
||||
byte level = SpatialPage.GetLevel(buffer);
|
||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||
uint parentId = SpatialPage.GetParentPageId(buffer);
|
||||
|
||||
// Collect all entries including the new one
|
||||
var entries = new List<(GeoBox Mbr, DocumentLocation Pointer)>();
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
SpatialPage.ReadEntry(buffer, i, out var m, out var p);
|
||||
entries.Add((m, p));
|
||||
}
|
||||
entries.Add((newMbr, newPointer));
|
||||
|
||||
// Pick Seeds
|
||||
PickSeeds(entries, out var seed1, out var seed2);
|
||||
entries.Remove(seed1);
|
||||
entries.Remove(seed2);
|
||||
|
||||
// Initialize two nodes
|
||||
uint newPageId = _storage.AllocatePage();
|
||||
SpatialPage.Initialize(buffer, pageId, isLeaf, level);
|
||||
SpatialPage.Initialize(newBuffer, newPageId, isLeaf, level);
|
||||
SpatialPage.SetParentPageId(buffer, parentId);
|
||||
SpatialPage.SetParentPageId(newBuffer, parentId);
|
||||
|
||||
SpatialPage.WriteEntry(buffer, 0, seed1.Mbr, seed1.Pointer);
|
||||
SpatialPage.SetEntryCount(buffer, 1);
|
||||
SpatialPage.WriteEntry(newBuffer, 0, seed2.Mbr, seed2.Pointer);
|
||||
SpatialPage.SetEntryCount(newBuffer, 1);
|
||||
|
||||
GeoBox mbr1 = seed1.Mbr;
|
||||
GeoBox mbr2 = seed2.Mbr;
|
||||
|
||||
// Distribute remaining entries
|
||||
while (entries.Count > 0)
|
||||
{
|
||||
var entry = entries[0];
|
||||
entries.RemoveAt(0);
|
||||
|
||||
var exp1 = mbr1.ExpandTo(entry.Mbr);
|
||||
var exp2 = mbr2.ExpandTo(entry.Mbr);
|
||||
double d1 = exp1.Area - mbr1.Area;
|
||||
double d2 = exp2.Area - mbr2.Area;
|
||||
|
||||
if (d1 < d2)
|
||||
{
|
||||
int idx = SpatialPage.GetEntryCount(buffer);
|
||||
SpatialPage.WriteEntry(buffer, idx, entry.Mbr, entry.Pointer);
|
||||
SpatialPage.SetEntryCount(buffer, (ushort)(idx + 1));
|
||||
mbr1 = exp1;
|
||||
}
|
||||
else
|
||||
{
|
||||
int idx = SpatialPage.GetEntryCount(newBuffer);
|
||||
SpatialPage.WriteEntry(newBuffer, idx, entry.Mbr, entry.Pointer);
|
||||
SpatialPage.SetEntryCount(newBuffer, (ushort)(idx + 1));
|
||||
mbr2 = exp2;
|
||||
}
|
||||
}
|
||||
|
||||
// Write pages
|
||||
if (transaction != null)
|
||||
{
|
||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||
_storage.WritePage(newPageId, transaction.TransactionId, newBuffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
_storage.WritePageImmediate(pageId, buffer);
|
||||
_storage.WritePageImmediate(newPageId, newBuffer);
|
||||
}
|
||||
|
||||
// Propagate split upwards
|
||||
if (pageId == _rootPageId)
|
||||
{
|
||||
// New Root
|
||||
uint newRootId = _storage.AllocatePage();
|
||||
SpatialPage.Initialize(buffer, newRootId, false, (byte)(level + 1));
|
||||
SpatialPage.WriteEntry(buffer, 0, mbr1, new DocumentLocation(pageId, 0));
|
||||
SpatialPage.WriteEntry(buffer, 1, mbr2, new DocumentLocation(newPageId, 0));
|
||||
SpatialPage.SetEntryCount(buffer, 2);
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(newRootId, transaction.TransactionId, buffer);
|
||||
else
|
||||
_storage.WritePageImmediate(newRootId, buffer);
|
||||
|
||||
_rootPageId = newRootId;
|
||||
|
||||
// Update parent pointers
|
||||
UpdateParentPointer(pageId, newRootId, transaction);
|
||||
UpdateParentPointer(newPageId, newRootId, transaction);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Insert second node into parent
|
||||
InsertIntoNode(parentId, mbr2, new DocumentLocation(newPageId, 0), transaction);
|
||||
UpdateMBRUpwards(pageId, transaction);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReturnPageBuffer(buffer);
|
||||
ReturnPageBuffer(newBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateParentPointer(uint pageId, uint parentId, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||
SpatialPage.SetParentPageId(buffer, parentId);
|
||||
if (transaction != null)
|
||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||
else
|
||||
_storage.WritePageImmediate(pageId, buffer);
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private void PickSeeds(List<(GeoBox Mbr, DocumentLocation Pointer)> entries, out (GeoBox Mbr, DocumentLocation Pointer) s1, out (GeoBox Mbr, DocumentLocation Pointer) s2)
|
||||
{
|
||||
double maxWaste = double.MinValue;
|
||||
s1 = entries[0];
|
||||
s2 = entries[1];
|
||||
|
||||
for (int i = 0; i < entries.Count; i++)
|
||||
{
|
||||
for (int j = i + 1; j < entries.Count; j++)
|
||||
{
|
||||
var combined = entries[i].Mbr.ExpandTo(entries[j].Mbr);
|
||||
double waste = combined.Area - entries[i].Mbr.Area - entries[j].Mbr.Area;
|
||||
if (waste > maxWaste)
|
||||
{
|
||||
maxWaste = waste;
|
||||
s1 = entries[i];
|
||||
s2 = entries[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] RentPageBuffer()
|
||||
{
|
||||
return ArrayPool<byte>.Shared.Rent(_pageSize);
|
||||
}
|
||||
|
||||
private void ReturnPageBuffer(byte[] buffer)
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by the index.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
77
src/CBDD.Core/Indexing/SpatialMath.cs
Executable file
77
src/CBDD.Core/Indexing/SpatialMath.cs
Executable file
@@ -0,0 +1,77 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
public static class SpatialMath
|
||||
{
|
||||
private const double EarthRadiusKm = 6371.0;
|
||||
|
||||
/// <summary>
|
||||
/// Calculates distance between two points on Earth using Haversine formula.
|
||||
/// Result in kilometers.
|
||||
/// </summary>
|
||||
/// <param name="p1">The first point.</param>
|
||||
/// <param name="p2">The second point.</param>
|
||||
/// <returns>The distance in kilometers.</returns>
|
||||
internal static double DistanceKm(GeoPoint p1, GeoPoint p2) => DistanceKm(p1.Latitude, p1.Longitude, p2.Latitude, p2.Longitude);
|
||||
|
||||
/// <summary>
|
||||
/// Calculates distance between two coordinates on Earth using Haversine formula.
|
||||
/// </summary>
|
||||
/// <param name="lat1">The latitude of the first point.</param>
|
||||
/// <param name="lon1">The longitude of the first point.</param>
|
||||
/// <param name="lat2">The latitude of the second point.</param>
|
||||
/// <param name="lon2">The longitude of the second point.</param>
|
||||
/// <returns>The distance in kilometers.</returns>
|
||||
public static double DistanceKm(double lat1, double lon1, double lat2, double lon2)
|
||||
{
|
||||
double dLat = ToRadians(lat2 - lat1);
|
||||
double dLon = ToRadians(lon2 - lon1);
|
||||
|
||||
double a = Math.Sin(dLat / 2) * Math.Sin(dLat / 2) +
|
||||
Math.Cos(ToRadians(lat1)) * Math.Cos(ToRadians(lat2)) *
|
||||
Math.Sin(dLon / 2) * Math.Sin(dLon / 2);
|
||||
|
||||
double c = 2 * Math.Atan2(Math.Sqrt(a), Math.Sqrt(1 - a));
|
||||
return EarthRadiusKm * c;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a Bounding Box (MBR) centered at a point with a given radius in km.
|
||||
/// </summary>
|
||||
/// <param name="center">The center point.</param>
|
||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||
/// <returns>The bounding box.</returns>
|
||||
internal static GeoBox BoundingBox(GeoPoint center, double radiusKm) => BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a bounding box from a coordinate and radius.
|
||||
/// </summary>
|
||||
/// <param name="lat">The center latitude.</param>
|
||||
/// <param name="lon">The center longitude.</param>
|
||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||
/// <returns>The bounding box.</returns>
|
||||
internal static GeoBox InternalBoundingBox(double lat, double lon, double radiusKm) => BoundingBox(lat, lon, radiusKm);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a bounding box centered at a coordinate with a given radius in kilometers.
|
||||
/// </summary>
|
||||
/// <param name="lat">The center latitude.</param>
|
||||
/// <param name="lon">The center longitude.</param>
|
||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||
/// <returns>The bounding box.</returns>
|
||||
internal static GeoBox BoundingBox(double lat, double lon, double radiusKm)
|
||||
{
|
||||
double dLat = ToDegrees(radiusKm / EarthRadiusKm);
|
||||
double dLon = ToDegrees(radiusKm / (EarthRadiusKm * Math.Cos(ToRadians(lat))));
|
||||
|
||||
return new GeoBox(
|
||||
lat - dLat,
|
||||
lon - dLon,
|
||||
lat + dLat,
|
||||
lon + dLon);
|
||||
}
|
||||
|
||||
private static double ToRadians(double degrees) => degrees * Math.PI / 180.0;
|
||||
private static double ToDegrees(double radians) => radians * 180.0 / Math.PI;
|
||||
}
|
||||
124
src/CBDD.Core/Indexing/VectorMath.cs
Executable file
124
src/CBDD.Core/Indexing/VectorMath.cs
Executable file
@@ -0,0 +1,124 @@
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Numerics;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Optimized vector math utilities using SIMD if available.
|
||||
/// </summary>
|
||||
public static class VectorMath
|
||||
{
|
||||
/// <summary>
|
||||
/// Computes vector distance according to the selected metric.
|
||||
/// </summary>
|
||||
/// <param name="v1">The first vector.</param>
|
||||
/// <param name="v2">The second vector.</param>
|
||||
/// <param name="metric">The metric used to compute distance.</param>
|
||||
/// <returns>The distance value for the selected metric.</returns>
|
||||
public static float Distance(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2, VectorMetric metric)
|
||||
{
|
||||
return metric switch
|
||||
{
|
||||
VectorMetric.Cosine => 1.0f - CosineSimilarity(v1, v2),
|
||||
VectorMetric.L2 => EuclideanDistanceSquared(v1, v2),
|
||||
VectorMetric.DotProduct => -DotProduct(v1, v2), // HNSW uses "distance" so smaller is better
|
||||
_ => throw new ArgumentException($"Unsupported metric: {metric}")
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes cosine similarity between two vectors.
|
||||
/// </summary>
|
||||
/// <param name="v1">The first vector.</param>
|
||||
/// <param name="v2">The second vector.</param>
|
||||
/// <returns>The cosine similarity in the range [-1, 1].</returns>
|
||||
public static float CosineSimilarity(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||
{
|
||||
float dot = DotProduct(v1, v2);
|
||||
float mag1 = DotProduct(v1, v1);
|
||||
float mag2 = DotProduct(v2, v2);
|
||||
|
||||
if (mag1 == 0 || mag2 == 0) return 0;
|
||||
return dot / (MathF.Sqrt(mag1) * MathF.Sqrt(mag2));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the dot product of two vectors.
|
||||
/// </summary>
|
||||
/// <param name="v1">The first vector.</param>
|
||||
/// <param name="v2">The second vector.</param>
|
||||
/// <returns>The dot product value.</returns>
|
||||
public static float DotProduct(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||
{
|
||||
if (v1.Length != v2.Length)
|
||||
throw new ArgumentException("Vectors must have same length");
|
||||
|
||||
float dot = 0;
|
||||
int i = 0;
|
||||
|
||||
// SIMD Optimization for .NET
|
||||
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
||||
{
|
||||
var vDot = Vector<float>.Zero;
|
||||
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
||||
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
||||
|
||||
foreach (var chunk in Enumerable.Range(0, v1Span.Length))
|
||||
{
|
||||
vDot += v1Span[chunk] * v2Span[chunk];
|
||||
}
|
||||
|
||||
dot = Vector.Dot(vDot, Vector<float>.One);
|
||||
i = v1Span.Length * Vector<float>.Count;
|
||||
}
|
||||
|
||||
// Remaining elements
|
||||
for (; i < v1.Length; i++)
|
||||
{
|
||||
dot += v1[i] * v2[i];
|
||||
}
|
||||
|
||||
return dot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes squared Euclidean distance between two vectors.
|
||||
/// </summary>
|
||||
/// <param name="v1">The first vector.</param>
|
||||
/// <param name="v2">The second vector.</param>
|
||||
/// <returns>The squared Euclidean distance.</returns>
|
||||
public static float EuclideanDistanceSquared(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||
{
|
||||
if (v1.Length != v2.Length)
|
||||
throw new ArgumentException("Vectors must have same length");
|
||||
|
||||
float dist = 0;
|
||||
int i = 0;
|
||||
|
||||
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
||||
{
|
||||
var vDist = Vector<float>.Zero;
|
||||
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
||||
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
||||
|
||||
foreach (var chunk in Enumerable.Range(0, v1Span.Length))
|
||||
{
|
||||
var diff = v1Span[chunk] - v2Span[chunk];
|
||||
vDist += diff * diff;
|
||||
}
|
||||
|
||||
dist = Vector.Dot(vDist, Vector<float>.One);
|
||||
i = v1Span.Length * Vector<float>.Count;
|
||||
}
|
||||
|
||||
for (; i < v1.Length; i++)
|
||||
{
|
||||
float diff = v1[i] - v2[i];
|
||||
dist += diff * diff;
|
||||
}
|
||||
|
||||
return dist;
|
||||
}
|
||||
}
|
||||
30
src/CBDD.Core/Indexing/VectorSearchExtensions.cs
Executable file
30
src/CBDD.Core/Indexing/VectorSearchExtensions.cs
Executable file
@@ -0,0 +1,30 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
public static class VectorSearchExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Performs a similarity search on a vector property.
|
||||
/// This method is a marker for the LINQ query provider and is optimized using HNSW indexes if available.
|
||||
/// </summary>
|
||||
/// <param name="vector">The vector property of the entity.</param>
|
||||
/// <param name="query">The query vector to compare against.</param>
|
||||
/// <param name="k">Number of nearest neighbors to return.</param>
|
||||
/// <returns>True if the document is part of the top-k results (always returns true when evaluated in memory for compilation purposes).</returns>
|
||||
public static bool VectorSearch(this float[] vector, float[] query, int k)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a similarity search on a collection of vector properties.
|
||||
/// Used for entities with multiple vectors per document.
|
||||
/// </summary>
|
||||
/// <param name="vectors">The vector collection of the entity.</param>
|
||||
/// <param name="query">The query vector to compare against.</param>
|
||||
/// <param name="k">Number of nearest neighbors to return.</param>
|
||||
/// <returns>True if the document is part of the top-k results (always returns true when evaluated in memory for compilation purposes).</returns>
|
||||
public static bool VectorSearch(this IEnumerable<float[]> vectors, float[] query, int k)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
379
src/CBDD.Core/Indexing/VectorSearchIndex.cs
Executable file
379
src/CBDD.Core/Indexing/VectorSearchIndex.cs
Executable file
@@ -0,0 +1,379 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// HNSW (Hierarchical Navigable Small World) index implementation.
|
||||
/// Handles multi-vector indexing and similarity searches.
|
||||
/// </summary>
|
||||
public sealed class VectorSearchIndex
|
||||
{
|
||||
private struct NodeReference
|
||||
{
|
||||
public uint PageId;
|
||||
public int NodeIndex;
|
||||
public int MaxLevel;
|
||||
}
|
||||
|
||||
private readonly IIndexStorage _storage;
|
||||
private readonly IndexOptions _options;
|
||||
private uint _rootPageId;
|
||||
private readonly Random _random = new(42);
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new vector search index.
|
||||
/// </summary>
|
||||
/// <param name="storage">The storage engine used by the index.</param>
|
||||
/// <param name="options">Index configuration options.</param>
|
||||
/// <param name="rootPageId">Optional existing root page identifier.</param>
|
||||
public VectorSearchIndex(StorageEngine storage, IndexOptions options, uint rootPageId = 0)
|
||||
: this((IStorageEngine)storage, options, rootPageId)
|
||||
{
|
||||
}
|
||||
|
||||
internal VectorSearchIndex(IIndexStorage storage, IndexOptions options, uint rootPageId = 0)
|
||||
{
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_options = options;
|
||||
_rootPageId = rootPageId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the root page identifier of the index.
|
||||
/// </summary>
|
||||
public uint RootPageId => _rootPageId;
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a vector and its document location into the index.
|
||||
/// </summary>
|
||||
/// <param name="vector">The vector values to index.</param>
|
||||
/// <param name="docLocation">The document location associated with the vector.</param>
|
||||
/// <param name="transaction">Optional transaction context.</param>
|
||||
public void Insert(float[] vector, DocumentLocation docLocation, ITransaction? transaction = null)
|
||||
{
|
||||
if (vector.Length != _options.Dimensions)
|
||||
throw new ArgumentException($"Vector dimension mismatch. Expected {_options.Dimensions}, got {vector.Length}");
|
||||
|
||||
// 1. Determine level for new node
|
||||
int targetLevel = GetRandomLevel();
|
||||
|
||||
// 2. If index is empty, create first page and first node
|
||||
if (_rootPageId == 0)
|
||||
{
|
||||
_rootPageId = CreateNewPage(transaction);
|
||||
var pageBuffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(_rootPageId, transaction?.TransactionId, pageBuffer);
|
||||
VectorPage.WriteNode(pageBuffer, 0, docLocation, targetLevel, vector, _options.Dimensions);
|
||||
VectorPage.IncrementNodeCount(pageBuffer); // Helper needs to be added or handled
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(_rootPageId, transaction.TransactionId, pageBuffer);
|
||||
else
|
||||
_storage.WritePageImmediate(_rootPageId, pageBuffer);
|
||||
}
|
||||
finally { ReturnPageBuffer(pageBuffer); }
|
||||
return;
|
||||
}
|
||||
|
||||
// HNSW Core logic
|
||||
// 3. Find current entry point
|
||||
var entryPoint = GetEntryPoint();
|
||||
var currentPoint = entryPoint;
|
||||
|
||||
// 4. Greedy search down to targetLevel+1
|
||||
for (int l = entryPoint.MaxLevel; l > targetLevel; l--)
|
||||
{
|
||||
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
||||
}
|
||||
|
||||
// 5. Create the new node
|
||||
var newNode = AllocateNode(vector, docLocation, targetLevel, transaction);
|
||||
|
||||
// 6. For each layer from targetLevel down to 0
|
||||
for (int l = Math.Min(targetLevel, entryPoint.MaxLevel); l >= 0; l--)
|
||||
{
|
||||
var neighbors = SearchLayer(currentPoint, vector, _options.EfConstruction, l, transaction);
|
||||
var selectedNeighbors = SelectNeighbors(neighbors, vector, _options.M, l, transaction);
|
||||
|
||||
foreach (var neighbor in selectedNeighbors)
|
||||
{
|
||||
AddBidirectionalLink(newNode, neighbor, l, transaction);
|
||||
}
|
||||
|
||||
// Move currentPoint down for next level if available
|
||||
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
||||
}
|
||||
|
||||
// 7. Update entry point if new node is higher
|
||||
if (targetLevel > entryPoint.MaxLevel)
|
||||
{
|
||||
UpdateEntryPoint(newNode, transaction);
|
||||
}
|
||||
}
|
||||
|
||||
private IEnumerable<NodeReference> SelectNeighbors(IEnumerable<NodeReference> candidates, float[] query, int m, int level, ITransaction? transaction)
|
||||
{
|
||||
// Simple heuristic: just take top M nearest.
|
||||
// HNSW Paper suggests more complex heuristic to maintain connectivity diversity.
|
||||
return candidates.Take(m);
|
||||
}
|
||||
|
||||
private void AddBidirectionalLink(NodeReference node1, NodeReference node2, int level, ITransaction? transaction)
|
||||
{
|
||||
Link(node1, node2, level, transaction);
|
||||
Link(node2, node1, level, transaction);
|
||||
}
|
||||
|
||||
private void Link(NodeReference from, NodeReference to, int level, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(from.PageId, transaction?.TransactionId, buffer);
|
||||
var links = VectorPage.GetLinksSpan(buffer, from.NodeIndex, level, _options.Dimensions, _options.M);
|
||||
|
||||
// Find first empty slot (PageId == 0)
|
||||
for (int i = 0; i < links.Length; i += 6)
|
||||
{
|
||||
var existing = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
||||
if (existing.PageId == 0)
|
||||
{
|
||||
new DocumentLocation(to.PageId, (ushort)to.NodeIndex).WriteTo(links.Slice(i, 6));
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(from.PageId, transaction.TransactionId, buffer);
|
||||
else
|
||||
_storage.WritePageImmediate(from.PageId, buffer);
|
||||
return;
|
||||
}
|
||||
}
|
||||
// If full, we should technically prune or redistribute links as per HNSW paper.
|
||||
// For now, we assume M is large enough or we skip (limited connectivity).
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private NodeReference AllocateNode(float[] vector, DocumentLocation docLoc, int level, ITransaction? transaction)
|
||||
{
|
||||
// Find a page with space or create new
|
||||
// For simplicity, we search for a page with available slots or append to a new one.
|
||||
// Implementation omitted for brevity but required for full persistence.
|
||||
uint pageId = _rootPageId; // Placeholder: need allocation strategy
|
||||
int index = 0;
|
||||
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||
index = VectorPage.GetNodeCount(buffer);
|
||||
VectorPage.WriteNode(buffer, index, docLoc, level, vector, _options.Dimensions);
|
||||
VectorPage.IncrementNodeCount(buffer);
|
||||
|
||||
if (transaction != null)
|
||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||
else
|
||||
_storage.WritePageImmediate(pageId, buffer);
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
|
||||
return new NodeReference { PageId = pageId, NodeIndex = index, MaxLevel = level };
|
||||
}
|
||||
|
||||
private void UpdateEntryPoint(NodeReference newEntry, ITransaction? transaction)
|
||||
{
|
||||
// Store in index header or collection metadata
|
||||
// For now, it's a simplification.
|
||||
}
|
||||
|
||||
private NodeReference GreedySearch(NodeReference entryPoint, float[] query, int level, ITransaction? transaction)
|
||||
{
|
||||
bool changed = true;
|
||||
var current = entryPoint;
|
||||
float currentDist = VectorMath.Distance(query, LoadVector(current, transaction), _options.Metric);
|
||||
|
||||
while (changed)
|
||||
{
|
||||
changed = false;
|
||||
foreach (var neighbor in GetNeighbors(current, level, transaction))
|
||||
{
|
||||
float dist = VectorMath.Distance(query, LoadVector(neighbor, transaction), _options.Metric);
|
||||
if (dist < currentDist)
|
||||
{
|
||||
currentDist = dist;
|
||||
current = neighbor;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private IEnumerable<NodeReference> SearchLayer(NodeReference entryPoint, float[] query, int ef, int level, ITransaction? transaction)
|
||||
{
|
||||
var visited = new HashSet<NodeReference>();
|
||||
var candidates = new PriorityQueue<NodeReference, float>();
|
||||
var result = new PriorityQueue<NodeReference, float>();
|
||||
|
||||
float dist = VectorMath.Distance(query, LoadVector(entryPoint, transaction), _options.Metric);
|
||||
candidates.Enqueue(entryPoint, dist);
|
||||
result.Enqueue(entryPoint, -dist); // Max-heap for results
|
||||
visited.Add(entryPoint);
|
||||
|
||||
while (candidates.Count > 0)
|
||||
{
|
||||
float d_c = 0;
|
||||
candidates.TryPeek(out var c, out d_c);
|
||||
result.TryPeek(out var f, out var d_f);
|
||||
|
||||
if (d_c > -d_f) break;
|
||||
|
||||
candidates.Dequeue();
|
||||
|
||||
foreach (var e in GetNeighbors(c, level, transaction))
|
||||
{
|
||||
if (!visited.Contains(e))
|
||||
{
|
||||
visited.Add(e);
|
||||
result.TryPeek(out _, out d_f);
|
||||
float d_e = VectorMath.Distance(query, LoadVector(e, transaction), _options.Metric);
|
||||
|
||||
if (d_e < -d_f || result.Count < ef)
|
||||
{
|
||||
candidates.Enqueue(e, d_e);
|
||||
result.Enqueue(e, -d_e);
|
||||
if (result.Count > ef) result.Dequeue();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert result to list (ordered by distance)
|
||||
var list = new List<NodeReference>();
|
||||
while (result.Count > 0) list.Add(result.Dequeue());
|
||||
list.Reverse();
|
||||
return list;
|
||||
}
|
||||
|
||||
private NodeReference GetEntryPoint()
|
||||
{
|
||||
// For now, assume a fixed location or track it in page 0 of index
|
||||
// TODO: Real implementation
|
||||
return new NodeReference { PageId = _rootPageId, NodeIndex = 0, MaxLevel = 0 };
|
||||
}
|
||||
|
||||
private float[] LoadVector(NodeReference node, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||
float[] vector = new float[_options.Dimensions];
|
||||
VectorPage.ReadNodeData(buffer, node.NodeIndex, out _, out _, vector);
|
||||
return vector;
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Searches the index for the nearest vectors to the query.
|
||||
/// </summary>
|
||||
/// <param name="query">The query vector.</param>
|
||||
/// <param name="k">The number of nearest results to return.</param>
|
||||
/// <param name="efSearch">The search breadth parameter.</param>
|
||||
/// <param name="transaction">Optional transaction context.</param>
|
||||
/// <returns>The nearest vector search results.</returns>
|
||||
public IEnumerable<VectorSearchResult> Search(float[] query, int k, int efSearch = 100, ITransaction? transaction = null)
|
||||
{
|
||||
if (_rootPageId == 0) yield break;
|
||||
|
||||
var entryPoint = GetEntryPoint();
|
||||
var currentPoint = entryPoint;
|
||||
|
||||
// 1. Greedy search through higher layers to find entry point for level 0
|
||||
for (int l = entryPoint.MaxLevel; l > 0; l--)
|
||||
{
|
||||
currentPoint = GreedySearch(currentPoint, query, l, transaction);
|
||||
}
|
||||
|
||||
// 2. Comprehensive search on level 0
|
||||
var nearest = SearchLayer(currentPoint, query, Math.Max(efSearch, k), 0, transaction);
|
||||
|
||||
// 3. Return top-k results
|
||||
int count = 0;
|
||||
foreach (var node in nearest)
|
||||
{
|
||||
if (count++ >= k) break;
|
||||
|
||||
float dist = VectorMath.Distance(query, LoadVector(node, transaction), _options.Metric);
|
||||
var loc = LoadDocumentLocation(node, transaction);
|
||||
yield return new VectorSearchResult(loc, dist);
|
||||
}
|
||||
}
|
||||
|
||||
private DocumentLocation LoadDocumentLocation(NodeReference node, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||
VectorPage.ReadNodeData(buffer, node.NodeIndex, out var loc, out _, new float[0]); // Vector not needed here
|
||||
return loc;
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private IEnumerable<NodeReference> GetNeighbors(NodeReference node, int level, ITransaction? transaction)
|
||||
{
|
||||
var buffer = RentPageBuffer();
|
||||
var results = new List<NodeReference>();
|
||||
try
|
||||
{
|
||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||
var links = VectorPage.GetLinksSpan(buffer, node.NodeIndex, level, _options.Dimensions, _options.M);
|
||||
|
||||
for (int i = 0; i < links.Length; i += 6)
|
||||
{
|
||||
var loc = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
||||
if (loc.PageId == 0) break; // End of links
|
||||
|
||||
results.Add(new NodeReference { PageId = loc.PageId, NodeIndex = loc.SlotIndex });
|
||||
}
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
return results;
|
||||
}
|
||||
|
||||
private int GetRandomLevel()
|
||||
{
|
||||
// Probability p = 1/M for each level
|
||||
double p = 1.0 / _options.M;
|
||||
int level = 0;
|
||||
while (_random.NextDouble() < p && level < 15)
|
||||
{
|
||||
level++;
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
private uint CreateNewPage(ITransaction? transaction)
|
||||
{
|
||||
uint pageId = _storage.AllocatePage();
|
||||
var buffer = RentPageBuffer();
|
||||
try
|
||||
{
|
||||
VectorPage.Initialize(buffer, pageId, _options.Dimensions, _options.M);
|
||||
_storage.WritePageImmediate(pageId, buffer);
|
||||
return pageId;
|
||||
}
|
||||
finally { ReturnPageBuffer(buffer); }
|
||||
}
|
||||
|
||||
private byte[] RentPageBuffer() => System.Buffers.ArrayPool<byte>.Shared.Rent(_storage.PageSize);
|
||||
private void ReturnPageBuffer(byte[] buffer) => System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
public record struct VectorSearchResult(DocumentLocation Location, float Distance);
|
||||
225
src/CBDD.Core/Metadata/EntityTypeBuilder.cs
Executable file
225
src/CBDD.Core/Metadata/EntityTypeBuilder.cs
Executable file
@@ -0,0 +1,225 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||
|
||||
public class EntityTypeBuilder<T> where T : class
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the configured collection name for the entity type.
|
||||
/// </summary>
|
||||
public string? CollectionName { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured indexes for the entity type.
|
||||
/// </summary>
|
||||
public List<IndexBuilder<T>> Indexes { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the primary key selector expression.
|
||||
/// </summary>
|
||||
public LambdaExpression? PrimaryKeySelector { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the primary key value is generated on add.
|
||||
/// </summary>
|
||||
public bool ValueGeneratedOnAdd { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured primary key property name.
|
||||
/// </summary>
|
||||
public string? PrimaryKeyName { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured property converter types keyed by property name.
|
||||
/// </summary>
|
||||
public Dictionary<string, Type> PropertyConverters { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Sets the collection name for the entity type.
|
||||
/// </summary>
|
||||
/// <param name="name">The collection name.</param>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> ToCollection(string name)
|
||||
{
|
||||
CollectionName = name;
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an index for the specified key selector.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The key type.</typeparam>
|
||||
/// <param name="keySelector">The key selector expression.</param>
|
||||
/// <param name="name">The optional index name.</param>
|
||||
/// <param name="unique">A value indicating whether the index is unique.</param>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> HasIndex<TKey>(Expression<Func<T, TKey>> keySelector, string? name = null, bool unique = false)
|
||||
{
|
||||
Indexes.Add(new IndexBuilder<T>(keySelector, name, unique));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a vector index for the specified key selector.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The key type.</typeparam>
|
||||
/// <param name="keySelector">The key selector expression.</param>
|
||||
/// <param name="dimensions">The vector dimensions.</param>
|
||||
/// <param name="metric">The vector similarity metric.</param>
|
||||
/// <param name="name">The optional index name.</param>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> HasVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector, int dimensions, VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
||||
{
|
||||
Indexes.Add(new IndexBuilder<T>(keySelector, name, false, IndexType.Vector, dimensions, metric));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a spatial index for the specified key selector.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The key type.</typeparam>
|
||||
/// <param name="keySelector">The key selector expression.</param>
|
||||
/// <param name="name">The optional index name.</param>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> HasSpatialIndex<TKey>(Expression<Func<T, TKey>> keySelector, string? name = null)
|
||||
{
|
||||
Indexes.Add(new IndexBuilder<T>(keySelector, name, false, IndexType.Spatial));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the primary key selector for the entity type.
|
||||
/// </summary>
|
||||
/// <typeparam name="TKey">The key type.</typeparam>
|
||||
/// <param name="keySelector">The primary key selector expression.</param>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> HasKey<TKey>(Expression<Func<T, TKey>> keySelector)
|
||||
{
|
||||
PrimaryKeySelector = keySelector;
|
||||
PrimaryKeyName = ExpressionAnalyzer.ExtractPropertyPaths(keySelector).FirstOrDefault() ?? "_id";
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configures a converter for the primary key property.
|
||||
/// </summary>
|
||||
/// <typeparam name="TConverter">The converter type.</typeparam>
|
||||
/// <returns>The current entity type builder.</returns>
|
||||
public EntityTypeBuilder<T> HasConversion<TConverter>()
|
||||
{
|
||||
if (!string.IsNullOrEmpty(PrimaryKeyName))
|
||||
{
|
||||
PropertyConverters[PrimaryKeyName] = typeof(TConverter);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configures a specific property on the entity type.
|
||||
/// </summary>
|
||||
/// <typeparam name="TProperty">The property type.</typeparam>
|
||||
/// <param name="propertyExpression">The property expression.</param>
|
||||
/// <returns>A builder for the selected property.</returns>
|
||||
public PropertyBuilder Property<TProperty>(Expression<Func<T, TProperty>> propertyExpression)
|
||||
{
|
||||
var propertyName = ExpressionAnalyzer.ExtractPropertyPaths(propertyExpression).FirstOrDefault();
|
||||
return new PropertyBuilder(this, propertyName);
|
||||
}
|
||||
|
||||
public class PropertyBuilder
|
||||
{
|
||||
private readonly EntityTypeBuilder<T> _parent;
|
||||
private readonly string? _propertyName;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="PropertyBuilder"/> class.
|
||||
/// </summary>
|
||||
/// <param name="parent">The parent entity type builder.</param>
|
||||
/// <param name="propertyName">The property name.</param>
|
||||
public PropertyBuilder(EntityTypeBuilder<T> parent, string? propertyName)
|
||||
{
|
||||
_parent = parent;
|
||||
_propertyName = propertyName;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the configured property as value generated on add.
|
||||
/// </summary>
|
||||
/// <returns>The current property builder.</returns>
|
||||
public PropertyBuilder ValueGeneratedOnAdd()
|
||||
{
|
||||
if (_propertyName == _parent.PrimaryKeyName)
|
||||
{
|
||||
_parent.ValueGeneratedOnAdd = true;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Configures a converter for the configured property.
|
||||
/// </summary>
|
||||
/// <typeparam name="TConverter">The converter type.</typeparam>
|
||||
/// <returns>The current property builder.</returns>
|
||||
public PropertyBuilder HasConversion<TConverter>()
|
||||
{
|
||||
if (!string.IsNullOrEmpty(_propertyName))
|
||||
{
|
||||
_parent.PropertyConverters[_propertyName] = typeof(TConverter);
|
||||
}
|
||||
return this;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class IndexBuilder<T>
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the index key selector expression.
|
||||
/// </summary>
|
||||
public LambdaExpression KeySelector { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured index name.
|
||||
/// </summary>
|
||||
public string? Name { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the index is unique.
|
||||
/// </summary>
|
||||
public bool IsUnique { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the index type.
|
||||
/// </summary>
|
||||
public IndexType Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the vector dimensions.
|
||||
/// </summary>
|
||||
public int Dimensions { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the vector metric.
|
||||
/// </summary>
|
||||
public VectorMetric Metric { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="IndexBuilder{T}"/> class.
|
||||
/// </summary>
|
||||
/// <param name="keySelector">The index key selector expression.</param>
|
||||
/// <param name="name">The optional index name.</param>
|
||||
/// <param name="unique">A value indicating whether the index is unique.</param>
|
||||
/// <param name="type">The index type.</param>
|
||||
/// <param name="dimensions">The vector dimensions.</param>
|
||||
/// <param name="metric">The vector metric.</param>
|
||||
public IndexBuilder(LambdaExpression keySelector, string? name, bool unique, IndexType type = IndexType.BTree, int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
||||
{
|
||||
KeySelector = keySelector;
|
||||
Name = name;
|
||||
IsUnique = unique;
|
||||
Type = type;
|
||||
Dimensions = dimensions;
|
||||
Metric = metric;
|
||||
}
|
||||
}
|
||||
30
src/CBDD.Core/Metadata/ModelBuilder.cs
Executable file
30
src/CBDD.Core/Metadata/ModelBuilder.cs
Executable file
@@ -0,0 +1,30 @@
|
||||
using System.Linq.Expressions;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||
|
||||
public class ModelBuilder
|
||||
{
|
||||
private readonly Dictionary<Type, object> _entityBuilders = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or creates the entity builder for the specified entity type.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The entity type.</typeparam>
|
||||
/// <returns>The entity builder for <typeparamref name="T"/>.</returns>
|
||||
public EntityTypeBuilder<T> Entity<T>() where T : class
|
||||
{
|
||||
if (!_entityBuilders.TryGetValue(typeof(T), out var builder))
|
||||
{
|
||||
builder = new EntityTypeBuilder<T>();
|
||||
_entityBuilders[typeof(T)] = builder;
|
||||
}
|
||||
return (EntityTypeBuilder<T>)builder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all registered entity builders.
|
||||
/// </summary>
|
||||
/// <returns>A read-only dictionary of entity builders keyed by entity type.</returns>
|
||||
public IReadOnlyDictionary<Type, object> GetEntityBuilders() => _entityBuilders;
|
||||
}
|
||||
20
src/CBDD.Core/Metadata/ValueConverter.cs
Executable file
20
src/CBDD.Core/Metadata/ValueConverter.cs
Executable file
@@ -0,0 +1,20 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a bidirectional conversion between a model type (e.g. ValueObject)
|
||||
/// and a provider type supported by the storage engine (e.g. string, int, Guid, ObjectId).
|
||||
/// </summary>
|
||||
public abstract class ValueConverter<TModel, TProvider>
|
||||
{
|
||||
/// <summary>
|
||||
/// Converts the model value to the provider value.
|
||||
/// </summary>
|
||||
/// <param name="model">The model value to convert.</param>
|
||||
public abstract TProvider ConvertToProvider(TModel model);
|
||||
|
||||
/// <summary>
|
||||
/// Converts the provider value back to the model value.
|
||||
/// </summary>
|
||||
/// <param name="provider">The provider value to convert.</param>
|
||||
public abstract TModel ConvertFromProvider(TProvider provider);
|
||||
}
|
||||
97
src/CBDD.Core/Query/BTreeExpressionVisitor.cs
Executable file
97
src/CBDD.Core/Query/BTreeExpressionVisitor.cs
Executable file
@@ -0,0 +1,97 @@
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal class BTreeExpressionVisitor : ExpressionVisitor
|
||||
{
|
||||
private readonly QueryModel _model = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the query model built while visiting an expression tree.
|
||||
/// </summary>
|
||||
public QueryModel GetModel() => _model;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Expression VisitMethodCall(MethodCallExpression node)
|
||||
{
|
||||
if (node.Method.DeclaringType == typeof(Queryable))
|
||||
{
|
||||
switch (node.Method.Name)
|
||||
{
|
||||
case "Where":
|
||||
VisitWhere(node);
|
||||
break;
|
||||
case "Select":
|
||||
VisitSelect(node);
|
||||
break;
|
||||
case "OrderBy":
|
||||
case "OrderByDescending":
|
||||
VisitOrderBy(node);
|
||||
break;
|
||||
case "Take":
|
||||
VisitTake(node);
|
||||
break;
|
||||
case "Skip":
|
||||
VisitSkip(node);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return base.VisitMethodCall(node);
|
||||
}
|
||||
|
||||
private void VisitWhere(MethodCallExpression node)
|
||||
{
|
||||
// Recursively visit source first (to preserve order or chained calls)
|
||||
Visit(node.Arguments[0]);
|
||||
|
||||
var predicate = (UnaryExpression)node.Arguments[1];
|
||||
var lambda = (LambdaExpression)predicate.Operand;
|
||||
|
||||
if (_model.WhereClause == null)
|
||||
{
|
||||
_model.WhereClause = lambda;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Combine predicates (AND)
|
||||
var parameter = Expression.Parameter(lambda.Parameters[0].Type, "x");
|
||||
var body = Expression.AndAlso(
|
||||
Expression.Invoke(_model.WhereClause, parameter),
|
||||
Expression.Invoke(lambda, parameter)
|
||||
);
|
||||
_model.WhereClause = Expression.Lambda(body, parameter);
|
||||
}
|
||||
}
|
||||
|
||||
private void VisitSelect(MethodCallExpression node)
|
||||
{
|
||||
Visit(node.Arguments[0]);
|
||||
var selector = (UnaryExpression)node.Arguments[1];
|
||||
_model.SelectClause = (LambdaExpression)selector.Operand;
|
||||
}
|
||||
|
||||
private void VisitOrderBy(MethodCallExpression node)
|
||||
{
|
||||
Visit(node.Arguments[0]);
|
||||
var keySelector = (UnaryExpression)node.Arguments[1];
|
||||
_model.OrderByClause = (LambdaExpression)keySelector.Operand;
|
||||
_model.OrderDescending = node.Method.Name == "OrderByDescending";
|
||||
}
|
||||
|
||||
private void VisitTake(MethodCallExpression node)
|
||||
{
|
||||
Visit(node.Arguments[0]);
|
||||
var countExpression = (ConstantExpression)node.Arguments[1];
|
||||
if (countExpression.Value != null)
|
||||
_model.Take = (int)countExpression.Value;
|
||||
}
|
||||
|
||||
private void VisitSkip(MethodCallExpression node)
|
||||
{
|
||||
Visit(node.Arguments[0]);
|
||||
var countExpression = (ConstantExpression)node.Arguments[1];
|
||||
if (countExpression.Value != null)
|
||||
_model.Skip = (int)countExpression.Value;
|
||||
}
|
||||
}
|
||||
173
src/CBDD.Core/Query/BTreeQueryProvider.cs
Executable file
173
src/CBDD.Core/Query/BTreeQueryProvider.cs
Executable file
@@ -0,0 +1,173 @@
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using static ZB.MOM.WW.CBDD.Core.Query.IndexOptimizer;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
||||
{
|
||||
private readonly DocumentCollection<TId, T> _collection;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="BTreeQueryProvider{TId, T}"/> class.
|
||||
/// </summary>
|
||||
/// <param name="collection">The backing document collection.</param>
|
||||
public BTreeQueryProvider(DocumentCollection<TId, T> collection)
|
||||
{
|
||||
_collection = collection;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a query from the specified expression.
|
||||
/// </summary>
|
||||
/// <param name="expression">The query expression.</param>
|
||||
/// <returns>An <see cref="IQueryable"/> representing the query.</returns>
|
||||
public IQueryable CreateQuery(Expression expression)
|
||||
{
|
||||
var elementType = expression.Type.GetGenericArguments()[0];
|
||||
try
|
||||
{
|
||||
return (IQueryable)Activator.CreateInstance(
|
||||
typeof(BTreeQueryable<>).MakeGenericType(elementType),
|
||||
new object[] { this, expression })!;
|
||||
}
|
||||
catch (TargetInvocationException ex)
|
||||
{
|
||||
throw ex.InnerException ?? ex;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a strongly typed query from the specified expression.
|
||||
/// </summary>
|
||||
/// <typeparam name="TElement">The element type of the query.</typeparam>
|
||||
/// <param name="expression">The query expression.</param>
|
||||
/// <returns>An <see cref="IQueryable{T}"/> representing the query.</returns>
|
||||
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
|
||||
{
|
||||
return new BTreeQueryable<TElement>(this, expression);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a query expression.
|
||||
/// </summary>
|
||||
/// <param name="expression">The query expression.</param>
|
||||
/// <returns>The query result.</returns>
|
||||
public object? Execute(Expression expression)
|
||||
{
|
||||
return Execute<object>(expression);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes a query expression and returns a strongly typed result.
|
||||
/// </summary>
|
||||
/// <typeparam name="TResult">The result type.</typeparam>
|
||||
/// <param name="expression">The query expression.</param>
|
||||
/// <returns>The query result.</returns>
|
||||
public TResult Execute<TResult>(Expression expression)
|
||||
{
|
||||
// 1. Visit to get model using strict BTreeExpressionVisitor (for optimization only)
|
||||
// We only care about WHERE clause for optimization.
|
||||
// GroupBy, Select, OrderBy, etc. are handled by EnumerableRewriter.
|
||||
|
||||
var visitor = new BTreeExpressionVisitor();
|
||||
visitor.Visit(expression);
|
||||
var model = visitor.GetModel();
|
||||
|
||||
// 2. Data Fetching Strategy (Optimized or Full Scan)
|
||||
IEnumerable<T> sourceData = null!;
|
||||
|
||||
// A. Try Index Optimization (Only if Where clause exists)
|
||||
var indexOpt = IndexOptimizer.TryOptimize<T>(model, _collection.GetIndexes());
|
||||
if (indexOpt != null)
|
||||
{
|
||||
if (indexOpt.IsVectorSearch)
|
||||
{
|
||||
sourceData = _collection.VectorSearch(indexOpt.IndexName, indexOpt.VectorQuery!, indexOpt.K);
|
||||
}
|
||||
else if (indexOpt.IsSpatialSearch)
|
||||
{
|
||||
sourceData = indexOpt.SpatialType == SpatialQueryType.Near
|
||||
? _collection.Near(indexOpt.IndexName, indexOpt.SpatialPoint, indexOpt.RadiusKm)
|
||||
: _collection.Within(indexOpt.IndexName, indexOpt.SpatialMin, indexOpt.SpatialMax);
|
||||
}
|
||||
else
|
||||
{
|
||||
sourceData = _collection.QueryIndex(indexOpt.IndexName, indexOpt.MinValue, indexOpt.MaxValue);
|
||||
}
|
||||
}
|
||||
|
||||
// B. Try Scan Optimization (if no index used)
|
||||
if (sourceData == null)
|
||||
{
|
||||
Func<ZB.MOM.WW.CBDD.Bson.BsonSpanReader, bool>? bsonPredicate = null;
|
||||
if (model.WhereClause != null)
|
||||
{
|
||||
bsonPredicate = BsonExpressionEvaluator.TryCompile<T>(model.WhereClause);
|
||||
}
|
||||
|
||||
if (bsonPredicate != null)
|
||||
{
|
||||
sourceData = _collection.Scan(bsonPredicate);
|
||||
}
|
||||
}
|
||||
|
||||
// C. Fallback to Full Scan
|
||||
if (sourceData == null)
|
||||
{
|
||||
sourceData = _collection.FindAll();
|
||||
}
|
||||
|
||||
// 3. Rewrite Expression Tree to use Enumerable
|
||||
// Replace the "Root" IQueryable with our sourceData IEnumerable
|
||||
|
||||
// We need to find the root IQueryable in the expression to replace it.
|
||||
// It's likely the first argument of the first method call, or a constant.
|
||||
|
||||
var rootFinder = new RootFinder();
|
||||
rootFinder.Visit(expression);
|
||||
var root = rootFinder.Root;
|
||||
|
||||
if (root == null) throw new InvalidOperationException("Could not find root Queryable in expression");
|
||||
|
||||
var rewriter = new EnumerableRewriter(root, sourceData);
|
||||
var rewrittenExpression = rewriter.Visit(expression);
|
||||
|
||||
// 4. Compile and Execute
|
||||
// The rewritten expression is now a tree of IEnumerable calls returning TResult.
|
||||
// We need to turn it into a Func<TResult> and invoke it.
|
||||
|
||||
if (rewrittenExpression.Type != typeof(TResult))
|
||||
{
|
||||
// If TResult is object (non-generic Execute), we need to cast
|
||||
rewrittenExpression = Expression.Convert(rewrittenExpression, typeof(TResult));
|
||||
}
|
||||
|
||||
var lambda = Expression.Lambda<Func<TResult>>(rewrittenExpression);
|
||||
var compiled = lambda.Compile();
|
||||
return compiled();
|
||||
}
|
||||
|
||||
private class RootFinder : ExpressionVisitor
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the root queryable found in the expression tree.
|
||||
/// </summary>
|
||||
public IQueryable? Root { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Expression VisitConstant(ConstantExpression node)
|
||||
{
|
||||
// If we found a Queryable, that's our root source
|
||||
if (Root == null && node.Value is IQueryable q)
|
||||
{
|
||||
// We typically want the "base" queryable (the BTreeQueryable instance)
|
||||
// In a chain like Coll.Where.Select, the root is Coll.
|
||||
Root = q;
|
||||
}
|
||||
return base.VisitConstant(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
56
src/CBDD.Core/Query/BTreeQueryable.cs
Executable file
56
src/CBDD.Core/Query/BTreeQueryable.cs
Executable file
@@ -0,0 +1,56 @@
|
||||
using System.Collections;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new queryable wrapper for the specified provider and expression.
|
||||
/// </summary>
|
||||
/// <param name="provider">The query provider.</param>
|
||||
/// <param name="expression">The expression tree.</param>
|
||||
public BTreeQueryable(IQueryProvider provider, Expression expression)
|
||||
{
|
||||
Provider = provider;
|
||||
Expression = expression;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new queryable wrapper for the specified provider.
|
||||
/// </summary>
|
||||
/// <param name="provider">The query provider.</param>
|
||||
public BTreeQueryable(IQueryProvider provider)
|
||||
{
|
||||
Provider = provider;
|
||||
Expression = Expression.Constant(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the element type returned by this query.
|
||||
/// </summary>
|
||||
public Type ElementType => typeof(T);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the expression tree associated with this query.
|
||||
/// </summary>
|
||||
public Expression Expression { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the query provider for this query.
|
||||
/// </summary>
|
||||
public IQueryProvider Provider { get; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
return Provider.Execute<IEnumerable<T>>(Expression).GetEnumerator();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
}
|
||||
143
src/CBDD.Core/Query/BsonExpressionEvaluator.cs
Executable file
143
src/CBDD.Core/Query/BsonExpressionEvaluator.cs
Executable file
@@ -0,0 +1,143 @@
|
||||
using System.Linq.Expressions;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal static class BsonExpressionEvaluator
|
||||
{
|
||||
/// <summary>
|
||||
/// Attempts to compile a LINQ predicate expression into a BSON reader predicate.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The entity type of the original expression.</typeparam>
|
||||
/// <param name="expression">The lambda expression to compile.</param>
|
||||
/// <returns>A compiled predicate when supported; otherwise, <see langword="null"/>.</returns>
|
||||
public static Func<BsonSpanReader, bool>? TryCompile<T>(LambdaExpression expression)
|
||||
{
|
||||
// Simple optimization for: x => x.Prop op Constant
|
||||
if (expression.Body is BinaryExpression binary)
|
||||
{
|
||||
var left = binary.Left;
|
||||
var right = binary.Right;
|
||||
var nodeType = binary.NodeType;
|
||||
|
||||
// Normalize: Ensure Property is on Left
|
||||
if (right is MemberExpression && left is ConstantExpression)
|
||||
{
|
||||
(left, right) = (right, left);
|
||||
// Flip operator
|
||||
nodeType = Flip(nodeType);
|
||||
}
|
||||
|
||||
if (left is MemberExpression member && right is ConstantExpression constant)
|
||||
{
|
||||
// Check if member is property of parameter
|
||||
if (member.Expression == expression.Parameters[0])
|
||||
{
|
||||
var propertyName = member.Member.Name.ToLowerInvariant();
|
||||
var value = constant.Value;
|
||||
|
||||
// Handle Id mapping?
|
||||
// If property is "id", Bson field is "_id"
|
||||
if (propertyName == "id") propertyName = "_id";
|
||||
|
||||
return CreatePredicate(propertyName, value, nodeType);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static ExpressionType Flip(ExpressionType type) => type switch
|
||||
{
|
||||
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
||||
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
||||
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
||||
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
||||
_ => type
|
||||
};
|
||||
|
||||
private static Func<BsonSpanReader, bool>? CreatePredicate(string propertyName, object? targetValue, ExpressionType op)
|
||||
{
|
||||
// We need to return a delegate that searches for propertyName in BsonSpanReader and compares
|
||||
|
||||
return reader =>
|
||||
{
|
||||
try
|
||||
{
|
||||
reader.ReadDocumentSize();
|
||||
while (reader.Remaining > 0)
|
||||
{
|
||||
var type = reader.ReadBsonType();
|
||||
if (type == 0) break;
|
||||
|
||||
var name = reader.ReadElementHeader();
|
||||
|
||||
if (name == propertyName)
|
||||
{
|
||||
// Found it! Read value and compare
|
||||
return Compare(ref reader, type, targetValue, op);
|
||||
}
|
||||
|
||||
reader.SkipValue(type);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return false; // Not found
|
||||
};
|
||||
}
|
||||
|
||||
private static bool Compare(ref BsonSpanReader reader, BsonType type, object? target, ExpressionType op)
|
||||
{
|
||||
// This is complex because we need to handle types.
|
||||
// For MVP, handle Int32, String, ObjectId
|
||||
|
||||
if (type == BsonType.Int32)
|
||||
{
|
||||
var val = reader.ReadInt32();
|
||||
if (target is int targetInt)
|
||||
{
|
||||
return op switch
|
||||
{
|
||||
ExpressionType.Equal => val == targetInt,
|
||||
ExpressionType.NotEqual => val != targetInt,
|
||||
ExpressionType.GreaterThan => val > targetInt,
|
||||
ExpressionType.GreaterThanOrEqual => val >= targetInt,
|
||||
ExpressionType.LessThan => val < targetInt,
|
||||
ExpressionType.LessThanOrEqual => val <= targetInt,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
}
|
||||
else if (type == BsonType.String)
|
||||
{
|
||||
var val = reader.ReadString();
|
||||
if (target is string targetStr)
|
||||
{
|
||||
var cmp = string.Compare(val, targetStr, StringComparison.Ordinal);
|
||||
return op switch
|
||||
{
|
||||
ExpressionType.Equal => cmp == 0,
|
||||
ExpressionType.NotEqual => cmp != 0,
|
||||
ExpressionType.GreaterThan => cmp > 0,
|
||||
ExpressionType.GreaterThanOrEqual => cmp >= 0,
|
||||
ExpressionType.LessThan => cmp < 0,
|
||||
ExpressionType.LessThanOrEqual => cmp <= 0,
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
}
|
||||
else if (type == BsonType.ObjectId && target is ObjectId targetId)
|
||||
{
|
||||
var val = reader.ReadObjectId();
|
||||
// ObjectId only supports Equal check easily unless we implement complex logic
|
||||
if (op == ExpressionType.Equal) return val.Equals(targetId);
|
||||
if (op == ExpressionType.NotEqual) return !val.Equals(targetId);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
86
src/CBDD.Core/Query/EnumerableRewriter.cs
Executable file
86
src/CBDD.Core/Query/EnumerableRewriter.cs
Executable file
@@ -0,0 +1,86 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal class EnumerableRewriter : ExpressionVisitor
|
||||
{
|
||||
private readonly IQueryable _source;
|
||||
private readonly object _target;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="EnumerableRewriter"/> class.
|
||||
/// </summary>
|
||||
/// <param name="source">The original queryable source to replace.</param>
|
||||
/// <param name="target">The target enumerable-backed object.</param>
|
||||
public EnumerableRewriter(IQueryable source, object target)
|
||||
{
|
||||
_source = source;
|
||||
_target = target;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Expression VisitConstant(ConstantExpression node)
|
||||
{
|
||||
// Replace the IQueryable source with the materialized IEnumerable
|
||||
if (node.Value == _source)
|
||||
{
|
||||
return Expression.Constant(_target);
|
||||
}
|
||||
return base.VisitConstant(node);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override Expression VisitMethodCall(MethodCallExpression node)
|
||||
{
|
||||
if (node.Method.DeclaringType == typeof(Queryable))
|
||||
{
|
||||
var methodName = node.Method.Name;
|
||||
var typeArgs = node.Method.GetGenericArguments();
|
||||
var args = new Expression[node.Arguments.Count];
|
||||
|
||||
for (int i = 0; i < node.Arguments.Count; i++)
|
||||
{
|
||||
var arg = Visit(node.Arguments[i]);
|
||||
|
||||
// Strip Quote from lambda arguments
|
||||
if (arg is UnaryExpression quote && quote.NodeType == ExpressionType.Quote)
|
||||
{
|
||||
var lambda = (LambdaExpression)quote.Operand;
|
||||
arg = Expression.Constant(lambda.Compile());
|
||||
}
|
||||
args[i] = arg;
|
||||
}
|
||||
|
||||
var enumerableMethods = typeof(Enumerable)
|
||||
.GetMethods(BindingFlags.Public | BindingFlags.Static)
|
||||
.Where(m => m.Name == methodName && m.GetGenericArguments().Length == typeArgs.Length);
|
||||
|
||||
foreach (var m in enumerableMethods)
|
||||
{
|
||||
var parameters = m.GetParameters();
|
||||
if (parameters.Length != args.Length) continue;
|
||||
|
||||
// Simple check: create generic method and see if it works?
|
||||
// Or check parameter compatibility properly.
|
||||
// For now, assume single match for standard LINQ operators (simplified)
|
||||
try
|
||||
{
|
||||
var genericMethod = m.MakeGenericMethod(typeArgs);
|
||||
// Check if arguments are assignable (basic check)
|
||||
// The first argument is usually "this IEnumerable<TSource>"
|
||||
return Expression.Call(genericMethod, args);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Ignore and try next overload
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return base.VisitMethodCall(node);
|
||||
}
|
||||
}
|
||||
299
src/CBDD.Core/Query/IndexOptimizer.cs
Executable file
299
src/CBDD.Core/Query/IndexOptimizer.cs
Executable file
@@ -0,0 +1,299 @@
|
||||
using System.Linq.Expressions;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal static class IndexOptimizer
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the selected index and bounds for an optimized query.
|
||||
/// </summary>
|
||||
public class OptimizationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the selected index name.
|
||||
/// </summary>
|
||||
public string IndexName { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the minimum bound value.
|
||||
/// </summary>
|
||||
public object? MinValue { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum bound value.
|
||||
/// </summary>
|
||||
public object? MaxValue { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the query uses a range.
|
||||
/// </summary>
|
||||
public bool IsRange { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the query uses vector search.
|
||||
/// </summary>
|
||||
public bool IsVectorSearch { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the vector query values.
|
||||
/// </summary>
|
||||
public float[]? VectorQuery { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of nearest neighbors for vector search.
|
||||
/// </summary>
|
||||
public int K { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the query uses spatial search.
|
||||
/// </summary>
|
||||
public bool IsSpatialSearch { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the center point for near queries.
|
||||
/// </summary>
|
||||
public (double Latitude, double Longitude) SpatialPoint { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the search radius in kilometers.
|
||||
/// </summary>
|
||||
public double RadiusKm { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the minimum point for within queries.
|
||||
/// </summary>
|
||||
public (double Latitude, double Longitude) SpatialMin { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum point for within queries.
|
||||
/// </summary>
|
||||
public (double Latitude, double Longitude) SpatialMax { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the spatial query type.
|
||||
/// </summary>
|
||||
public SpatialQueryType SpatialType { get; set; }
|
||||
}
|
||||
|
||||
public enum SpatialQueryType { Near, Within }
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to optimize a query model using available indexes.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">The document type.</typeparam>
|
||||
/// <param name="model">The query model.</param>
|
||||
/// <param name="indexes">The available collection indexes.</param>
|
||||
/// <returns>An optimization result when optimization is possible; otherwise, <see langword="null"/>.</returns>
|
||||
public static OptimizationResult? TryOptimize<T>(QueryModel model, IEnumerable<CollectionIndexInfo> indexes)
|
||||
{
|
||||
if (model.WhereClause == null) return null;
|
||||
|
||||
return OptimizeExpression(model.WhereClause.Body, model.WhereClause.Parameters[0], indexes);
|
||||
}
|
||||
|
||||
private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable<CollectionIndexInfo> indexes)
|
||||
{
|
||||
// ... (Existing AndAlso logic remains the same) ...
|
||||
if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso)
|
||||
{
|
||||
var left = OptimizeExpression(binary.Left, parameter, indexes);
|
||||
var right = OptimizeExpression(binary.Right, parameter, indexes);
|
||||
|
||||
if (left != null && right != null && left.IndexName == right.IndexName)
|
||||
{
|
||||
return new OptimizationResult
|
||||
{
|
||||
IndexName = left.IndexName,
|
||||
MinValue = left.MinValue ?? right.MinValue,
|
||||
MaxValue = left.MaxValue ?? right.MaxValue,
|
||||
IsRange = true
|
||||
};
|
||||
}
|
||||
return left ?? right;
|
||||
}
|
||||
|
||||
// Handle Simple Binary Predicates
|
||||
var (propertyName, value, op) = ParseSimplePredicate(expression, parameter);
|
||||
if (propertyName != null)
|
||||
{
|
||||
var index = indexes.FirstOrDefault(i => Matches(i, propertyName));
|
||||
if (index != null)
|
||||
{
|
||||
var result = new OptimizationResult { IndexName = index.Name };
|
||||
switch (op)
|
||||
{
|
||||
case ExpressionType.Equal:
|
||||
result.MinValue = value;
|
||||
result.MaxValue = value;
|
||||
result.IsRange = false;
|
||||
break;
|
||||
case ExpressionType.GreaterThan:
|
||||
case ExpressionType.GreaterThanOrEqual:
|
||||
result.MinValue = value;
|
||||
result.MaxValue = null;
|
||||
result.IsRange = true;
|
||||
break;
|
||||
case ExpressionType.LessThan:
|
||||
case ExpressionType.LessThanOrEqual:
|
||||
result.MinValue = null;
|
||||
result.MaxValue = value;
|
||||
result.IsRange = true;
|
||||
break;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle StartsWith
|
||||
if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member)
|
||||
{
|
||||
if (member.Expression == parameter && call.Arguments[0] is ConstantExpression constant && constant.Value is string prefix)
|
||||
{
|
||||
var index = indexes.FirstOrDefault(i => Matches(i, member.Member.Name));
|
||||
if (index != null && index.Type == IndexType.BTree)
|
||||
{
|
||||
var nextPrefix = IncrementPrefix(prefix);
|
||||
return new OptimizationResult
|
||||
{
|
||||
IndexName = index.Name,
|
||||
MinValue = prefix,
|
||||
MaxValue = nextPrefix,
|
||||
IsRange = true
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Handle Method Calls (VectorSearch, Near, Within)
|
||||
if (expression is MethodCallExpression mcall)
|
||||
{
|
||||
// VectorSearch(this float[] vector, float[] query, int k)
|
||||
if (mcall.Method.Name == "VectorSearch" && mcall.Arguments[0] is MemberExpression vMember && vMember.Expression == parameter)
|
||||
{
|
||||
var query = EvaluateExpression<float[]>(mcall.Arguments[1]);
|
||||
var k = EvaluateExpression<int>(mcall.Arguments[2]);
|
||||
|
||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Vector && Matches(i, vMember.Member.Name));
|
||||
if (index != null)
|
||||
{
|
||||
return new OptimizationResult
|
||||
{
|
||||
IndexName = index.Name,
|
||||
IsVectorSearch = true,
|
||||
VectorQuery = query,
|
||||
K = k
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Near(this (double, double) point, (double, double) center, double radiusKm)
|
||||
if (mcall.Method.Name == "Near" && mcall.Arguments[0] is MemberExpression nMember && nMember.Expression == parameter)
|
||||
{
|
||||
var center = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
||||
var radius = EvaluateExpression<double>(mcall.Arguments[2]);
|
||||
|
||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, nMember.Member.Name));
|
||||
if (index != null)
|
||||
{
|
||||
return new OptimizationResult
|
||||
{
|
||||
IndexName = index.Name,
|
||||
IsSpatialSearch = true,
|
||||
SpatialType = SpatialQueryType.Near,
|
||||
SpatialPoint = center,
|
||||
RadiusKm = radius
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Within(this (double, double) point, (double, double) min, (double, double) max)
|
||||
if (mcall.Method.Name == "Within" && mcall.Arguments[0] is MemberExpression wMember && wMember.Expression == parameter)
|
||||
{
|
||||
var min = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
||||
var max = EvaluateExpression<(double, double)>(mcall.Arguments[2]);
|
||||
|
||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, wMember.Member.Name));
|
||||
if (index != null)
|
||||
{
|
||||
return new OptimizationResult
|
||||
{
|
||||
IndexName = index.Name,
|
||||
IsSpatialSearch = true,
|
||||
SpatialType = SpatialQueryType.Within,
|
||||
SpatialMin = min,
|
||||
SpatialMax = max
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string IncrementPrefix(string prefix)
|
||||
{
|
||||
if (string.IsNullOrEmpty(prefix)) return null!;
|
||||
char lastChar = prefix[prefix.Length - 1];
|
||||
if (lastChar == char.MaxValue) return prefix; // Cannot increment
|
||||
return prefix.Substring(0, prefix.Length - 1) + (char)(lastChar + 1);
|
||||
}
|
||||
|
||||
private static T EvaluateExpression<T>(Expression expression)
|
||||
{
|
||||
if (expression is ConstantExpression constant)
|
||||
{
|
||||
return (T)constant.Value!;
|
||||
}
|
||||
|
||||
// Evaluate more complex expressions (closures, properties, etc.)
|
||||
var lambda = Expression.Lambda(expression);
|
||||
var compiled = lambda.Compile();
|
||||
return (T)compiled.DynamicInvoke()!;
|
||||
}
|
||||
|
||||
private static bool Matches(CollectionIndexInfo index, string propertyName)
|
||||
{
|
||||
if (index.PropertyPaths == null || index.PropertyPaths.Length == 0) return false;
|
||||
return index.PropertyPaths[0].Equals(propertyName, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter)
|
||||
{
|
||||
if (expression is BinaryExpression binary)
|
||||
{
|
||||
var left = binary.Left;
|
||||
var right = binary.Right;
|
||||
var nodeType = binary.NodeType;
|
||||
|
||||
if (right is MemberExpression && left is ConstantExpression)
|
||||
{
|
||||
(left, right) = (right, left);
|
||||
nodeType = Flip(nodeType);
|
||||
}
|
||||
|
||||
if (left is MemberExpression member && right is ConstantExpression constant)
|
||||
{
|
||||
if (member.Expression == parameter)
|
||||
return (member.Member.Name, constant.Value, nodeType);
|
||||
}
|
||||
|
||||
// Handle Convert
|
||||
if (left is UnaryExpression unary && unary.Operand is MemberExpression member2 && right is ConstantExpression constant2)
|
||||
{
|
||||
if (member2.Expression == parameter)
|
||||
return (member2.Member.Name, constant2.Value, nodeType);
|
||||
}
|
||||
}
|
||||
return (null, null, ExpressionType.Default);
|
||||
}
|
||||
|
||||
private static ExpressionType Flip(ExpressionType type) => type switch
|
||||
{
|
||||
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
||||
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
||||
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
||||
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
||||
_ => type
|
||||
};
|
||||
}
|
||||
36
src/CBDD.Core/Query/QueryModel.cs
Executable file
36
src/CBDD.Core/Query/QueryModel.cs
Executable file
@@ -0,0 +1,36 @@
|
||||
using System.Linq.Expressions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||
|
||||
internal class QueryModel
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the filter expression.
|
||||
/// </summary>
|
||||
public LambdaExpression? WhereClause { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the projection expression.
|
||||
/// </summary>
|
||||
public LambdaExpression? SelectClause { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the ordering expression.
|
||||
/// </summary>
|
||||
public LambdaExpression? OrderByClause { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum number of results to return.
|
||||
/// </summary>
|
||||
public int? Take { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the number of results to skip.
|
||||
/// </summary>
|
||||
public int? Skip { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether ordering is descending.
|
||||
/// </summary>
|
||||
public bool OrderDescending { get; set; }
|
||||
}
|
||||
283
src/CBDD.Core/Storage/DictionaryPage.cs
Executable file
283
src/CBDD.Core/Storage/DictionaryPage.cs
Executable file
@@ -0,0 +1,283 @@
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using ZB.MOM.WW.CBDD.Core;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Page for storing dictionary entries (Key -> Value map).
|
||||
/// Uses a sorted list of keys for binary search within the page.
|
||||
/// Supports chaining via PageHeader.NextPageId for dictionaries larger than one page.
|
||||
/// </summary>
|
||||
public struct DictionaryPage
|
||||
{
|
||||
// Layout:
|
||||
// [PageHeader (32)]
|
||||
// [Count (2)]
|
||||
// [FreeSpaceEnd (2)]
|
||||
// [Offsets (Count * 2)] ...
|
||||
// ... Free Space ...
|
||||
// ... Data (Growing Downwards) ...
|
||||
|
||||
private const int HeaderSize = 32;
|
||||
private const int CountOffset = 32;
|
||||
private const int FreeSpaceEndOffset = 34;
|
||||
private const int OffsetsStart = 36;
|
||||
|
||||
/// <summary>
|
||||
/// Values 0-100 are reserved for internal system keys (e.g. _id, _v).
|
||||
/// </summary>
|
||||
public const ushort ReservedValuesEnd = 100;
|
||||
|
||||
/// <summary>
|
||||
/// Initialize a new dictionary page
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer to initialize.</param>
|
||||
/// <param name="pageId">The page identifier.</param>
|
||||
public static void Initialize(Span<byte> page, uint pageId)
|
||||
{
|
||||
// 1. Write Page Header
|
||||
var header = new PageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Dictionary,
|
||||
FreeBytes = (ushort)(page.Length - OffsetsStart),
|
||||
NextPageId = 0,
|
||||
TransactionId = 0,
|
||||
Checksum = 0
|
||||
};
|
||||
header.WriteTo(page);
|
||||
|
||||
// 2. Initialize Counts
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), 0);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), (ushort)page.Length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts a key-value pair into the page.
|
||||
/// Returns false if there is not enough space.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="key">The dictionary key.</param>
|
||||
/// <param name="value">The value mapped to the key.</param>
|
||||
/// <returns><see langword="true"/> if the entry was inserted; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool Insert(Span<byte> page, string key, ushort value)
|
||||
{
|
||||
var keyByteCount = Encoding.UTF8.GetByteCount(key);
|
||||
if (keyByteCount > 255) throw new ArgumentException("Key length must be <= 255 bytes");
|
||||
|
||||
// Entry Size: KeyLen(1) + Key(N) + Value(2)
|
||||
var entrySize = 1 + keyByteCount + 2;
|
||||
var requiredSpace = entrySize + 2; // +2 for Offset entry
|
||||
|
||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||
var freeSpaceEnd = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(FreeSpaceEndOffset));
|
||||
|
||||
var offsetsEnd = OffsetsStart + (count * 2);
|
||||
var freeSpace = freeSpaceEnd - offsetsEnd;
|
||||
|
||||
if (freeSpace < requiredSpace)
|
||||
{
|
||||
return false; // Page Full
|
||||
}
|
||||
|
||||
// 1. Prepare Data
|
||||
var insertionOffset = (ushort)(freeSpaceEnd - entrySize);
|
||||
page[insertionOffset] = (byte)keyByteCount; // Write Key Length
|
||||
Encoding.UTF8.GetBytes(key, page.Slice(insertionOffset + 1, keyByteCount)); // Write Key
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(insertionOffset + 1 + keyByteCount), value); // Write Value
|
||||
|
||||
// 2. Insert Offset into Sorted List
|
||||
// Find insert Index using spans
|
||||
ReadOnlySpan<byte> keyBytes = page.Slice(insertionOffset + 1, keyByteCount);
|
||||
int insertIndex = FindInsertIndex(page, count, keyBytes);
|
||||
|
||||
// Shift offsets if needed
|
||||
if (insertIndex < count)
|
||||
{
|
||||
var src = page.Slice(OffsetsStart + (insertIndex * 2), (count - insertIndex) * 2);
|
||||
var dest = page.Slice(OffsetsStart + ((insertIndex + 1) * 2));
|
||||
src.CopyTo(dest);
|
||||
}
|
||||
|
||||
// Write new offset
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(OffsetsStart + (insertIndex * 2)), insertionOffset);
|
||||
|
||||
// 3. Update Metadata
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), (ushort)(count + 1));
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), insertionOffset);
|
||||
|
||||
// Update FreeBytes in header (approximate)
|
||||
var pageHeader = PageHeader.ReadFrom(page);
|
||||
pageHeader.FreeBytes = (ushort)(insertionOffset - (OffsetsStart + ((count + 1) * 2)));
|
||||
pageHeader.WriteTo(page);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to find a value for the given key in THIS page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="keyBytes">The UTF-8 encoded key bytes.</param>
|
||||
/// <param name="value">When this method returns, contains the found value.</param>
|
||||
/// <returns><see langword="true"/> if the key was found; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool TryFind(ReadOnlySpan<byte> page, ReadOnlySpan<byte> keyBytes, out ushort value)
|
||||
{
|
||||
value = 0;
|
||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||
if (count == 0) return false;
|
||||
|
||||
// Binary Search
|
||||
int low = 0;
|
||||
int high = count - 1;
|
||||
|
||||
while (low <= high)
|
||||
{
|
||||
int mid = low + (high - low) / 2;
|
||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (mid * 2)));
|
||||
|
||||
// Read Key at Offset
|
||||
var keyLen = page[offset];
|
||||
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
||||
|
||||
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
||||
|
||||
if (comparison == 0)
|
||||
{
|
||||
value = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (comparison < 0)
|
||||
low = mid + 1;
|
||||
else
|
||||
high = mid - 1;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tries to find a value for the given key across a chain of DictionaryPages.
|
||||
/// </summary>
|
||||
/// <param name="storage">The storage engine used to read pages.</param>
|
||||
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
||||
/// <param name="key">The key to search for.</param>
|
||||
/// <param name="value">When this method returns, contains the found value.</param>
|
||||
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
||||
/// <returns><see langword="true"/> if the key was found; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool TryFindGlobal(StorageEngine storage, uint startPageId, string key, out ushort value, ulong? transactionId = null)
|
||||
{
|
||||
var keyByteCount = Encoding.UTF8.GetByteCount(key);
|
||||
Span<byte> keyBytes = keyByteCount <= 256 ? stackalloc byte[keyByteCount] : new byte[keyByteCount];
|
||||
Encoding.UTF8.GetBytes(key, keyBytes);
|
||||
|
||||
var pageId = startPageId;
|
||||
var pageBuffer = System.Buffers.ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||
try
|
||||
{
|
||||
while (pageId != 0)
|
||||
{
|
||||
// Read page
|
||||
storage.ReadPage(pageId, transactionId, pageBuffer);
|
||||
|
||||
// TryFind in this page
|
||||
if (TryFind(pageBuffer, keyBytes, out value))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Move to next page
|
||||
var header = PageHeader.ReadFrom(pageBuffer);
|
||||
pageId = header.NextPageId;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(pageBuffer);
|
||||
}
|
||||
|
||||
value = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
private static int FindInsertIndex(ReadOnlySpan<byte> page, int count, ReadOnlySpan<byte> keyBytes)
|
||||
{
|
||||
int low = 0;
|
||||
int high = count - 1;
|
||||
|
||||
while (low <= high)
|
||||
{
|
||||
int mid = low + (high - low) / 2;
|
||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (mid * 2)));
|
||||
|
||||
var keyLen = page[offset];
|
||||
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
||||
|
||||
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
||||
|
||||
if (comparison == 0) return mid;
|
||||
if (comparison < 0)
|
||||
low = mid + 1;
|
||||
else
|
||||
high = mid - 1;
|
||||
}
|
||||
return low;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all entries in the page (for debugging/dumping)
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>All key-value pairs in the page.</returns>
|
||||
public static IEnumerable<(string Key, ushort Value)> GetAll(ReadOnlySpan<byte> page)
|
||||
{
|
||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||
var list = new List<(string Key, ushort Value)>();
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (i * 2)));
|
||||
var keyLen = page[offset];
|
||||
var keyStr = Encoding.UTF8.GetString(page.Slice(offset + 1, keyLen));
|
||||
var val = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
||||
list.Add((keyStr, val));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
/// <summary>
|
||||
/// Retrieves all key-value pairs across a chain of DictionaryPages.
|
||||
/// Used for rebuilding the in-memory cache.
|
||||
/// </summary>
|
||||
/// <param name="storage">The storage engine used to read pages.</param>
|
||||
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
||||
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
||||
/// <returns>All key-value pairs across the page chain.</returns>
|
||||
public static IEnumerable<(string Key, ushort Value)> FindAllGlobal(StorageEngine storage, uint startPageId, ulong? transactionId = null)
|
||||
{
|
||||
var pageId = startPageId;
|
||||
var pageBuffer = System.Buffers.ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||
try
|
||||
{
|
||||
while (pageId != 0)
|
||||
{
|
||||
// Read page
|
||||
storage.ReadPage(pageId, transactionId, pageBuffer);
|
||||
|
||||
// Get all entries in this page
|
||||
foreach (var entry in GetAll(pageBuffer))
|
||||
{
|
||||
yield return entry;
|
||||
}
|
||||
|
||||
// Move to next page
|
||||
var header = PageHeader.ReadFrom(pageBuffer);
|
||||
pageId = header.NextPageId;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(pageBuffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
14
src/CBDD.Core/Storage/IIndexStorage.cs
Normal file
14
src/CBDD.Core/Storage/IIndexStorage.cs
Normal file
@@ -0,0 +1,14 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Narrow storage port for index structures (page operations + allocation only).
|
||||
/// </summary>
|
||||
internal interface IIndexStorage
|
||||
{
|
||||
int PageSize { get; }
|
||||
uint AllocatePage();
|
||||
void FreePage(uint pageId);
|
||||
void ReadPage(uint pageId, ulong? transactionId, Span<byte> destination);
|
||||
void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data);
|
||||
void WritePageImmediate(uint pageId, ReadOnlySpan<byte> data);
|
||||
}
|
||||
38
src/CBDD.Core/Storage/IStorageEngine.cs
Normal file
38
src/CBDD.Core/Storage/IStorageEngine.cs
Normal file
@@ -0,0 +1,38 @@
|
||||
using System.Collections.Concurrent;
|
||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
using ZB.MOM.WW.CBDD.Core.CDC;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Storage port used by collection/index orchestration to avoid concrete engine coupling.
|
||||
/// </summary>
|
||||
internal interface IStorageEngine : IIndexStorage, IDisposable
|
||||
{
|
||||
uint PageCount { get; }
|
||||
ChangeStreamDispatcher? Cdc { get; }
|
||||
CompressionOptions CompressionOptions { get; }
|
||||
CompressionService CompressionService { get; }
|
||||
CompressionTelemetry CompressionTelemetry { get; }
|
||||
|
||||
bool IsPageLocked(uint pageId, ulong excludingTxId);
|
||||
void RegisterCdc(ChangeStreamDispatcher cdc);
|
||||
|
||||
Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted);
|
||||
Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, CancellationToken ct = default);
|
||||
|
||||
CollectionMetadata? GetCollectionMetadata(string name);
|
||||
void SaveCollectionMetadata(CollectionMetadata metadata);
|
||||
void RegisterMappers(IEnumerable<IDocumentMapper> mappers);
|
||||
|
||||
List<BsonSchema> GetSchemas(uint rootPageId);
|
||||
uint AppendSchema(uint rootPageId, BsonSchema schema);
|
||||
|
||||
ConcurrentDictionary<string, ushort> GetKeyMap();
|
||||
ConcurrentDictionary<ushort, string> GetKeyReverseMap();
|
||||
ushort GetOrAddDictionaryEntry(string key);
|
||||
void RegisterKeys(IEnumerable<string> keys);
|
||||
}
|
||||
1003
src/CBDD.Core/Storage/PageFile.cs
Executable file
1003
src/CBDD.Core/Storage/PageFile.cs
Executable file
File diff suppressed because it is too large
Load Diff
75
src/CBDD.Core/Storage/PageHeader.cs
Executable file
75
src/CBDD.Core/Storage/PageHeader.cs
Executable file
@@ -0,0 +1,75 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a page header in the database file.
|
||||
/// Fixed 32-byte structure at the start of each page.
|
||||
/// Implemented as struct for efficient memory layout.
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||
public struct PageHeader
|
||||
{
|
||||
/// <summary>Page ID (offset in pages from start of file)</summary>
|
||||
[FieldOffset(0)]
|
||||
public uint PageId;
|
||||
|
||||
/// <summary>Type of this page</summary>
|
||||
[FieldOffset(4)]
|
||||
public PageType PageType;
|
||||
|
||||
/// <summary>Number of free bytes in this page</summary>
|
||||
[FieldOffset(5)]
|
||||
public ushort FreeBytes;
|
||||
|
||||
/// <summary>ID of next page in linked list (0 if none). For Page 0 (Header), this points to the First Free Page.</summary>
|
||||
[FieldOffset(7)]
|
||||
public uint NextPageId;
|
||||
|
||||
/// <summary>Transaction ID that last modified this page</summary>
|
||||
[FieldOffset(11)]
|
||||
public ulong TransactionId;
|
||||
|
||||
/// <summary>Checksum for data integrity (CRC32)</summary>
|
||||
[FieldOffset(19)]
|
||||
public uint Checksum;
|
||||
|
||||
/// <summary>Dictionary Root Page ID (Only used in Page 0 / File Header)</summary>
|
||||
[FieldOffset(23)]
|
||||
public uint DictionaryRootPageId;
|
||||
|
||||
[FieldOffset(27)]
|
||||
private byte _reserved5;
|
||||
[FieldOffset(28)]
|
||||
private byte _reserved6;
|
||||
[FieldOffset(29)]
|
||||
private byte _reserved7;
|
||||
[FieldOffset(30)]
|
||||
private byte _reserved8;
|
||||
[FieldOffset(31)]
|
||||
private byte _reserved9;
|
||||
|
||||
/// <summary>
|
||||
/// Writes the header to a span
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span that receives the serialized header.</param>
|
||||
public readonly void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < 32)
|
||||
throw new ArgumentException("Destination must be at least 32 bytes");
|
||||
|
||||
MemoryMarshal.Write(destination, in this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads a header from a span
|
||||
/// </summary>
|
||||
/// <param name="source">The source span containing a serialized header.</param>
|
||||
public static PageHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < 32)
|
||||
throw new ArgumentException("Source must be at least 32 bytes");
|
||||
|
||||
return MemoryMarshal.Read<PageHeader>(source);
|
||||
}
|
||||
}
|
||||
43
src/CBDD.Core/Storage/PageType.cs
Executable file
43
src/CBDD.Core/Storage/PageType.cs
Executable file
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Page types in the database file
|
||||
/// </summary>
|
||||
public enum PageType : byte
|
||||
{
|
||||
/// <summary>Empty/free page</summary>
|
||||
Empty = 0,
|
||||
|
||||
/// <summary>File header page (page 0)</summary>
|
||||
Header = 1,
|
||||
|
||||
/// <summary>Collection metadata page</summary>
|
||||
Collection = 2,
|
||||
|
||||
/// <summary>Data page containing documents</summary>
|
||||
Data = 3,
|
||||
|
||||
/// <summary>Index B+Tree node page</summary>
|
||||
Index = 4,
|
||||
|
||||
/// <summary>Free page list</summary>
|
||||
FreeList = 5,
|
||||
|
||||
/// <summary>Overflow page for large documents</summary>
|
||||
Overflow = 6,
|
||||
|
||||
/// <summary>Page marked as free/reusable</summary>
|
||||
Free = 10,
|
||||
|
||||
/// <summary>Dictionary page for string interning</summary>
|
||||
Dictionary = 7,
|
||||
|
||||
/// <summary>Schema versioning page</summary>
|
||||
Schema = 8,
|
||||
|
||||
/// <summary>HNSW Vector index page</summary>
|
||||
Vector = 9,
|
||||
|
||||
/// <summary>GEO Spatial index page</summary>
|
||||
Spatial = 11
|
||||
}
|
||||
208
src/CBDD.Core/Storage/SlottedPage.cs
Executable file
208
src/CBDD.Core/Storage/SlottedPage.cs
Executable file
@@ -0,0 +1,208 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Header for slotted pages supporting multiple variable-size documents per page.
|
||||
/// Fixed 24-byte structure at start of each data page.
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 24)]
|
||||
public struct SlottedPageHeader
|
||||
{
|
||||
/// <summary>Page ID</summary>
|
||||
[FieldOffset(0)]
|
||||
public uint PageId;
|
||||
|
||||
/// <summary>Type of page (Data, Overflow, Index, Metadata)</summary>
|
||||
[FieldOffset(4)]
|
||||
public PageType PageType;
|
||||
|
||||
/// <summary>Number of slot entries in this page</summary>
|
||||
[FieldOffset(8)]
|
||||
public ushort SlotCount;
|
||||
|
||||
/// <summary>Offset where free space starts (grows down with slots)</summary>
|
||||
[FieldOffset(10)]
|
||||
public ushort FreeSpaceStart;
|
||||
|
||||
/// <summary>Offset where free space ends (grows up with data)</summary>
|
||||
[FieldOffset(12)]
|
||||
public ushort FreeSpaceEnd;
|
||||
|
||||
/// <summary>Next overflow page ID (0 if none)</summary>
|
||||
[FieldOffset(14)]
|
||||
public uint NextOverflowPage;
|
||||
|
||||
/// <summary>Transaction ID that last modified this page</summary>
|
||||
[FieldOffset(18)]
|
||||
public uint TransactionId;
|
||||
|
||||
/// <summary>Reserved for future use</summary>
|
||||
[FieldOffset(22)]
|
||||
public ushort Reserved;
|
||||
|
||||
public const int Size = 24;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a header with the current slotted-page format marker.
|
||||
/// </summary>
|
||||
public SlottedPageHeader()
|
||||
{
|
||||
this = default;
|
||||
Reserved = StorageFormatConstants.SlottedPageFormatMarker;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets available free space in bytes
|
||||
/// </summary>
|
||||
public readonly int AvailableFreeSpace => FreeSpaceEnd - FreeSpaceStart;
|
||||
|
||||
/// <summary>
|
||||
/// Writes header to span
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span that receives the serialized header.</param>
|
||||
public readonly void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < Size)
|
||||
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
||||
|
||||
MemoryMarshal.Write(destination, in this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads header from span
|
||||
/// </summary>
|
||||
/// <param name="source">The source span containing the serialized header.</param>
|
||||
public static SlottedPageHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < Size)
|
||||
throw new ArgumentException($"Source must be at least {Size} bytes");
|
||||
|
||||
return MemoryMarshal.Read<SlottedPageHeader>(source);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Slot entry pointing to a document within a page.
|
||||
/// Fixed 8-byte structure in slot array.
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 8)]
|
||||
public struct SlotEntry
|
||||
{
|
||||
/// <summary>Offset to document data within page</summary>
|
||||
[FieldOffset(0)]
|
||||
public ushort Offset;
|
||||
|
||||
/// <summary>Length of document data in bytes</summary>
|
||||
[FieldOffset(2)]
|
||||
public ushort Length;
|
||||
|
||||
/// <summary>Slot flags (deleted, overflow, etc.)</summary>
|
||||
[FieldOffset(4)]
|
||||
public SlotFlags Flags;
|
||||
|
||||
public const int Size = 8;
|
||||
|
||||
/// <summary>
|
||||
/// Writes slot entry to span
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span that receives the serialized slot entry.</param>
|
||||
public readonly void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < Size)
|
||||
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
||||
|
||||
MemoryMarshal.Write(destination, in this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads slot entry from span
|
||||
/// </summary>
|
||||
/// <param name="source">The source span containing the serialized slot entry.</param>
|
||||
public static SlotEntry ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < Size)
|
||||
throw new ArgumentException($"Source must be at least {Size} bytes");
|
||||
|
||||
return MemoryMarshal.Read<SlotEntry>(source);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flags for slot entries
|
||||
/// </summary>
|
||||
[Flags]
|
||||
public enum SlotFlags : uint
|
||||
{
|
||||
/// <summary>Slot is active and contains data</summary>
|
||||
None = 0,
|
||||
|
||||
/// <summary>Slot is marked as deleted (can be reused)</summary>
|
||||
Deleted = 1 << 0,
|
||||
|
||||
/// <summary>Document continues in overflow pages</summary>
|
||||
HasOverflow = 1 << 1,
|
||||
|
||||
/// <summary>Document data is compressed</summary>
|
||||
Compressed = 1 << 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Location of a document within the database.
|
||||
/// Maps ObjectId to specific page and slot.
|
||||
/// </summary>
|
||||
public readonly struct DocumentLocation
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the page identifier containing the document.
|
||||
/// </summary>
|
||||
public uint PageId { get; init; }
|
||||
/// <summary>
|
||||
/// Gets the slot index within the page.
|
||||
/// </summary>
|
||||
public ushort SlotIndex { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="DocumentLocation"/> struct.
|
||||
/// </summary>
|
||||
/// <param name="pageId">The page identifier containing the document.</param>
|
||||
/// <param name="slotIndex">The slot index within the page.</param>
|
||||
public DocumentLocation(uint pageId, ushort slotIndex)
|
||||
{
|
||||
PageId = pageId;
|
||||
SlotIndex = slotIndex;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Serializes DocumentLocation to a byte span (6 bytes: 4 for PageId + 2 for SlotIndex)
|
||||
/// </summary>
|
||||
/// <param name="destination">The destination span that receives the serialized value.</param>
|
||||
public void WriteTo(Span<byte> destination)
|
||||
{
|
||||
if (destination.Length < 6)
|
||||
throw new ArgumentException("Destination must be at least 6 bytes", nameof(destination));
|
||||
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(destination, PageId);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(destination.Slice(4), SlotIndex);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserializes DocumentLocation from a byte span (6 bytes)
|
||||
/// </summary>
|
||||
/// <param name="source">The source span containing the serialized value.</param>
|
||||
public static DocumentLocation ReadFrom(ReadOnlySpan<byte> source)
|
||||
{
|
||||
if (source.Length < 6)
|
||||
throw new ArgumentException("Source must be at least 6 bytes", nameof(source));
|
||||
|
||||
var pageId = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(source);
|
||||
var slotIndex = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(source.Slice(4));
|
||||
|
||||
return new DocumentLocation(pageId, slotIndex);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Size in bytes when serialized
|
||||
/// </summary>
|
||||
public const int SerializedSize = 6;
|
||||
}
|
||||
165
src/CBDD.Core/Storage/SpatialPage.cs
Executable file
165
src/CBDD.Core/Storage/SpatialPage.cs
Executable file
@@ -0,0 +1,165 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Runtime.InteropServices;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Page for storing R-Tree nodes for Geospatial Indexing.
|
||||
/// </summary>
|
||||
internal struct SpatialPage
|
||||
{
|
||||
// Layout:
|
||||
// [PageHeader (32)]
|
||||
// [IsLeaf (1)]
|
||||
// [Level (1)]
|
||||
// [EntryCount (2)]
|
||||
// [ParentPageId (4)]
|
||||
// [Padding (8)]
|
||||
// [Entries (Contiguous)...]
|
||||
//
|
||||
// Each Entry: [MBR (4 * 8 = 32)] [Pointer (6)] = 38 bytes
|
||||
|
||||
private const int IsLeafOffset = 32;
|
||||
private const int LevelOffset = 33;
|
||||
private const int EntryCountOffset = 34;
|
||||
private const int ParentPageIdOffset = 36;
|
||||
private const int DataOffset = 48;
|
||||
|
||||
public const int EntrySize = 38; // 32 (GeoBox) + 6 (Pointer)
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a spatial page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer to initialize.</param>
|
||||
/// <param name="pageId">The page identifier.</param>
|
||||
/// <param name="isLeaf">Whether this page is a leaf node.</param>
|
||||
/// <param name="level">The tree level for this page.</param>
|
||||
public static void Initialize(Span<byte> page, uint pageId, bool isLeaf, byte level)
|
||||
{
|
||||
var header = new PageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Spatial,
|
||||
FreeBytes = (ushort)(page.Length - DataOffset),
|
||||
NextPageId = 0,
|
||||
TransactionId = 0
|
||||
};
|
||||
header.WriteTo(page);
|
||||
|
||||
page[IsLeafOffset] = (byte)(isLeaf ? 1 : 0);
|
||||
page[LevelOffset] = level;
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(EntryCountOffset), 0);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the page is a leaf node.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns><see langword="true"/> if the page is a leaf node; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool GetIsLeaf(ReadOnlySpan<byte> page) => page[IsLeafOffset] == 1;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the tree level stored in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The level value.</returns>
|
||||
public static byte GetLevel(ReadOnlySpan<byte> page) => page[LevelOffset];
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of entries in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The number of entries.</returns>
|
||||
public static ushort GetEntryCount(ReadOnlySpan<byte> page) => BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(EntryCountOffset));
|
||||
|
||||
/// <summary>
|
||||
/// Sets the number of entries in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="count">The entry count to set.</param>
|
||||
public static void SetEntryCount(Span<byte> page, ushort count) => BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(EntryCountOffset), count);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the parent page identifier.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The parent page identifier.</returns>
|
||||
public static uint GetParentPageId(ReadOnlySpan<byte> page) => BinaryPrimitives.ReadUInt32LittleEndian(page.Slice(ParentPageIdOffset));
|
||||
|
||||
/// <summary>
|
||||
/// Sets the parent page identifier.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="parentId">The parent page identifier.</param>
|
||||
public static void SetParentPageId(Span<byte> page, uint parentId) => BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), parentId);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum number of entries that can fit in a page.
|
||||
/// </summary>
|
||||
/// <param name="pageSize">The page size in bytes.</param>
|
||||
/// <returns>The maximum number of entries.</returns>
|
||||
public static int GetMaxEntries(int pageSize) => (pageSize - DataOffset) / EntrySize;
|
||||
|
||||
/// <summary>
|
||||
/// Writes an entry at the specified index.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="index">The entry index.</param>
|
||||
/// <param name="mbr">The minimum bounding rectangle for the entry.</param>
|
||||
/// <param name="pointer">The document location pointer.</param>
|
||||
public static void WriteEntry(Span<byte> page, int index, GeoBox mbr, DocumentLocation pointer)
|
||||
{
|
||||
int offset = DataOffset + (index * EntrySize);
|
||||
var entrySpan = page.Slice(offset, EntrySize);
|
||||
|
||||
// Write MBR (4 doubles)
|
||||
var doubles = MemoryMarshal.Cast<byte, double>(entrySpan.Slice(0, 32));
|
||||
doubles[0] = mbr.MinLat;
|
||||
doubles[1] = mbr.MinLon;
|
||||
doubles[2] = mbr.MaxLat;
|
||||
doubles[3] = mbr.MaxLon;
|
||||
|
||||
// Write Pointer (6 bytes)
|
||||
pointer.WriteTo(entrySpan.Slice(32, 6));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads an entry at the specified index.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="index">The entry index.</param>
|
||||
/// <param name="mbr">When this method returns, contains the entry MBR.</param>
|
||||
/// <param name="pointer">When this method returns, contains the entry document location.</param>
|
||||
public static void ReadEntry(ReadOnlySpan<byte> page, int index, out GeoBox mbr, out DocumentLocation pointer)
|
||||
{
|
||||
int offset = DataOffset + (index * EntrySize);
|
||||
var entrySpan = page.Slice(offset, EntrySize);
|
||||
|
||||
var doubles = MemoryMarshal.Cast<byte, double>(entrySpan.Slice(0, 32));
|
||||
mbr = new GeoBox(doubles[0], doubles[1], doubles[2], doubles[3]);
|
||||
pointer = DocumentLocation.ReadFrom(entrySpan.Slice(32, 6));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the combined MBR of all entries in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The combined MBR, or <see cref="GeoBox.Empty"/> when the page has no entries.</returns>
|
||||
public static GeoBox CalculateMBR(ReadOnlySpan<byte> page)
|
||||
{
|
||||
ushort count = GetEntryCount(page);
|
||||
if (count == 0) return GeoBox.Empty;
|
||||
|
||||
GeoBox result = GeoBox.Empty;
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
ReadEntry(page, i, out var mbr, out _);
|
||||
if (i == 0) result = mbr;
|
||||
else result = result.ExpandTo(mbr);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
287
src/CBDD.Core/Storage/StorageEngine.Collections.cs
Executable file
287
src/CBDD.Core/Storage/StorageEngine.Collections.cs
Executable file
@@ -0,0 +1,287 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public class CollectionMetadata
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the collection name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the root page identifier of the primary index.
|
||||
/// </summary>
|
||||
public uint PrimaryRootPageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the root page identifier of the schema chain.
|
||||
/// </summary>
|
||||
public uint SchemaRootPageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the collection index metadata list.
|
||||
/// </summary>
|
||||
public List<IndexMetadata> Indexes { get; } = new();
|
||||
}
|
||||
|
||||
public class IndexMetadata
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the index name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether this index enforces uniqueness.
|
||||
/// </summary>
|
||||
public bool IsUnique { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the index type.
|
||||
/// </summary>
|
||||
public IndexType Type { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets indexed property paths.
|
||||
/// </summary>
|
||||
public string[] PropertyPaths { get; set; } = Array.Empty<string>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets vector dimensions for vector indexes.
|
||||
/// </summary>
|
||||
public int Dimensions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the vector similarity metric for vector indexes.
|
||||
/// </summary>
|
||||
public VectorMetric Metric { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the root page identifier of the index structure.
|
||||
/// </summary>
|
||||
public uint RootPageId { get; set; }
|
||||
}
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets collection metadata by name.
|
||||
/// </summary>
|
||||
/// <param name="name">The collection name.</param>
|
||||
/// <returns>The collection metadata if found; otherwise, null.</returns>
|
||||
public CollectionMetadata? GetCollectionMetadata(string name)
|
||||
{
|
||||
return GetAllCollectionMetadata()
|
||||
.FirstOrDefault(x => string.Equals(x.Name, name, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all collection metadata entries currently registered in page 1.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionMetadata> GetAllCollectionMetadata()
|
||||
{
|
||||
var result = new List<CollectionMetadata>();
|
||||
var buffer = new byte[PageSize];
|
||||
ReadPage(1, null, buffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
if (header.PageType != PageType.Collection || header.SlotCount == 0)
|
||||
return result;
|
||||
|
||||
for (ushort i = 0; i < header.SlotCount; i++)
|
||||
{
|
||||
var slotOffset = SlottedPageHeader.Size + (i * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
continue;
|
||||
|
||||
if (slot.Offset < SlottedPageHeader.Size || slot.Offset + slot.Length > buffer.Length)
|
||||
continue;
|
||||
|
||||
if (TryDeserializeCollectionMetadata(buffer.AsSpan(slot.Offset, slot.Length), out var metadata) && metadata != null)
|
||||
{
|
||||
result.Add(metadata);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Saves collection metadata to the metadata page.
|
||||
/// </summary>
|
||||
/// <param name="metadata">The metadata to save.</param>
|
||||
public void SaveCollectionMetadata(CollectionMetadata metadata)
|
||||
{
|
||||
using var stream = new MemoryStream();
|
||||
using var writer = new BinaryWriter(stream);
|
||||
|
||||
writer.Write(metadata.Name);
|
||||
writer.Write(metadata.PrimaryRootPageId);
|
||||
writer.Write(metadata.SchemaRootPageId);
|
||||
writer.Write(metadata.Indexes.Count);
|
||||
foreach (var idx in metadata.Indexes)
|
||||
{
|
||||
writer.Write(idx.Name);
|
||||
writer.Write(idx.IsUnique);
|
||||
writer.Write((byte)idx.Type);
|
||||
writer.Write(idx.RootPageId);
|
||||
writer.Write(idx.PropertyPaths.Length);
|
||||
foreach (var path in idx.PropertyPaths)
|
||||
{
|
||||
writer.Write(path);
|
||||
}
|
||||
|
||||
if (idx.Type == IndexType.Vector)
|
||||
{
|
||||
writer.Write(idx.Dimensions);
|
||||
writer.Write((byte)idx.Metric);
|
||||
}
|
||||
}
|
||||
|
||||
var newData = stream.ToArray();
|
||||
|
||||
var buffer = new byte[PageSize];
|
||||
ReadPage(1, null, buffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
int existingSlotIndex = -1;
|
||||
|
||||
for (ushort i = 0; i < header.SlotCount; i++)
|
||||
{
|
||||
var slotOffset = SlottedPageHeader.Size + (i * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0) continue;
|
||||
|
||||
try
|
||||
{
|
||||
using var ms = new MemoryStream(buffer, slot.Offset, slot.Length, false);
|
||||
using var reader = new BinaryReader(ms);
|
||||
var name = reader.ReadString();
|
||||
|
||||
if (string.Equals(name, metadata.Name, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
existingSlotIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
|
||||
if (existingSlotIndex >= 0)
|
||||
{
|
||||
var slotOffset = SlottedPageHeader.Size + (existingSlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||
slot.Flags |= SlotFlags.Deleted;
|
||||
slot.WriteTo(buffer.AsSpan(slotOffset));
|
||||
}
|
||||
|
||||
if (header.AvailableFreeSpace < newData.Length + SlotEntry.Size)
|
||||
{
|
||||
// Compact logic omitted as per current architecture
|
||||
throw new InvalidOperationException("Not enough space in Metadata Page (Page 1) to save collection metadata.");
|
||||
}
|
||||
|
||||
int docOffset = header.FreeSpaceEnd - newData.Length;
|
||||
newData.CopyTo(buffer.AsSpan(docOffset));
|
||||
|
||||
ushort slotIndex;
|
||||
if (existingSlotIndex >= 0)
|
||||
{
|
||||
slotIndex = (ushort)existingSlotIndex;
|
||||
}
|
||||
else
|
||||
{
|
||||
slotIndex = header.SlotCount;
|
||||
header.SlotCount++;
|
||||
}
|
||||
|
||||
var newSlotEntryOffset = SlottedPageHeader.Size + (slotIndex * SlotEntry.Size);
|
||||
var newSlot = new SlotEntry
|
||||
{
|
||||
Offset = (ushort)docOffset,
|
||||
Length = (ushort)newData.Length,
|
||||
Flags = SlotFlags.None
|
||||
};
|
||||
newSlot.WriteTo(buffer.AsSpan(newSlotEntryOffset));
|
||||
|
||||
header.FreeSpaceEnd = (ushort)docOffset;
|
||||
if (existingSlotIndex == -1)
|
||||
{
|
||||
header.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (header.SlotCount * SlotEntry.Size));
|
||||
}
|
||||
|
||||
header.WriteTo(buffer);
|
||||
WritePageImmediate(1, buffer);
|
||||
}
|
||||
|
||||
private static bool TryDeserializeCollectionMetadata(ReadOnlySpan<byte> rawBytes, out CollectionMetadata? metadata)
|
||||
{
|
||||
metadata = null;
|
||||
|
||||
try
|
||||
{
|
||||
using var ms = new MemoryStream(rawBytes.ToArray());
|
||||
using var reader = new BinaryReader(ms);
|
||||
|
||||
var collName = reader.ReadString();
|
||||
var parsed = new CollectionMetadata { Name = collName };
|
||||
parsed.PrimaryRootPageId = reader.ReadUInt32();
|
||||
parsed.SchemaRootPageId = reader.ReadUInt32();
|
||||
|
||||
var indexCount = reader.ReadInt32();
|
||||
if (indexCount < 0)
|
||||
return false;
|
||||
|
||||
for (int j = 0; j < indexCount; j++)
|
||||
{
|
||||
var idx = new IndexMetadata
|
||||
{
|
||||
Name = reader.ReadString(),
|
||||
IsUnique = reader.ReadBoolean(),
|
||||
Type = (IndexType)reader.ReadByte(),
|
||||
RootPageId = reader.ReadUInt32()
|
||||
};
|
||||
|
||||
var pathCount = reader.ReadInt32();
|
||||
if (pathCount < 0)
|
||||
return false;
|
||||
|
||||
idx.PropertyPaths = new string[pathCount];
|
||||
for (int k = 0; k < pathCount; k++)
|
||||
idx.PropertyPaths[k] = reader.ReadString();
|
||||
|
||||
if (idx.Type == IndexType.Vector)
|
||||
{
|
||||
idx.Dimensions = reader.ReadInt32();
|
||||
idx.Metric = (VectorMetric)reader.ReadByte();
|
||||
}
|
||||
|
||||
parsed.Indexes.Add(idx);
|
||||
}
|
||||
|
||||
metadata = parsed;
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers all BSON keys used by a set of mappers into the global dictionary.
|
||||
/// </summary>
|
||||
/// <param name="mappers">The mappers whose keys should be registered.</param>
|
||||
public void RegisterMappers(IEnumerable<IDocumentMapper> mappers)
|
||||
{
|
||||
var allKeys = mappers.SelectMany(m => m.UsedKeys).Distinct();
|
||||
RegisterKeys(allKeys);
|
||||
}
|
||||
}
|
||||
448
src/CBDD.Core/Storage/StorageEngine.Diagnostics.cs
Normal file
448
src/CBDD.Core/Storage/StorageEngine.Diagnostics.cs
Normal file
@@ -0,0 +1,448 @@
|
||||
using System.Buffers.Binary;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Aggregated page counts grouped by page type.
|
||||
/// </summary>
|
||||
public sealed class PageTypeUsageEntry
|
||||
{
|
||||
public PageType PageType { get; init; }
|
||||
public int PageCount { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-collection page usage summary.
|
||||
/// </summary>
|
||||
public sealed class CollectionPageUsageEntry
|
||||
{
|
||||
public string CollectionName { get; init; } = string.Empty;
|
||||
public int TotalDistinctPages { get; init; }
|
||||
public int DataPages { get; init; }
|
||||
public int OverflowPages { get; init; }
|
||||
public int IndexPages { get; init; }
|
||||
public int OtherPages { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-collection compression ratio summary.
|
||||
/// </summary>
|
||||
public sealed class CollectionCompressionRatioEntry
|
||||
{
|
||||
public string CollectionName { get; init; } = string.Empty;
|
||||
public long DocumentCount { get; init; }
|
||||
public long CompressedDocumentCount { get; init; }
|
||||
public long BytesBeforeCompression { get; init; }
|
||||
public long BytesAfterCompression { get; init; }
|
||||
public double CompressionRatio => BytesAfterCompression <= 0 ? 1.0 : (double)BytesBeforeCompression / BytesAfterCompression;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summary of free-list and reclaimable tail information.
|
||||
/// </summary>
|
||||
public sealed class FreeListSummary
|
||||
{
|
||||
public uint PageCount { get; init; }
|
||||
public int FreePageCount { get; init; }
|
||||
public long FreeBytes { get; init; }
|
||||
public double FragmentationPercent { get; init; }
|
||||
public uint TailReclaimablePages { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single page entry in fragmentation reporting.
|
||||
/// </summary>
|
||||
public sealed class FragmentationPageEntry
|
||||
{
|
||||
public uint PageId { get; init; }
|
||||
public PageType PageType { get; init; }
|
||||
public bool IsFreePage { get; init; }
|
||||
public int FreeBytes { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detailed fragmentation map and totals.
|
||||
/// </summary>
|
||||
public sealed class FragmentationMapReport
|
||||
{
|
||||
public IReadOnlyList<FragmentationPageEntry> Pages { get; init; } = Array.Empty<FragmentationPageEntry>();
|
||||
public long TotalFreeBytes { get; init; }
|
||||
public double FragmentationPercent { get; init; }
|
||||
public uint TailReclaimablePages { get; init; }
|
||||
}
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets page usage grouped by page type.
|
||||
/// </summary>
|
||||
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
||||
{
|
||||
var pageCount = _pageFile.NextPageId;
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var counts = new Dictionary<PageType, int>();
|
||||
|
||||
for (uint pageId = 0; pageId < pageCount; pageId++)
|
||||
{
|
||||
_pageFile.ReadPage(pageId, buffer);
|
||||
var pageType = PageHeader.ReadFrom(buffer).PageType;
|
||||
counts[pageType] = counts.TryGetValue(pageType, out var count) ? count + 1 : 1;
|
||||
}
|
||||
|
||||
return counts
|
||||
.OrderBy(x => (byte)x.Key)
|
||||
.Select(x => new PageTypeUsageEntry
|
||||
{
|
||||
PageType = x.Key,
|
||||
PageCount = x.Value
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection page usage by resolving primary-index locations and related index roots.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
||||
{
|
||||
var metadataEntries = GetAllCollectionMetadata();
|
||||
var results = new List<CollectionPageUsageEntry>(metadataEntries.Count);
|
||||
|
||||
foreach (var metadata in metadataEntries)
|
||||
{
|
||||
var pageIds = new HashSet<uint>();
|
||||
|
||||
if (metadata.PrimaryRootPageId != 0)
|
||||
pageIds.Add(metadata.PrimaryRootPageId);
|
||||
if (metadata.SchemaRootPageId != 0)
|
||||
pageIds.Add(metadata.SchemaRootPageId);
|
||||
|
||||
foreach (var indexMetadata in metadata.Indexes)
|
||||
{
|
||||
if (indexMetadata.RootPageId != 0)
|
||||
pageIds.Add(indexMetadata.RootPageId);
|
||||
}
|
||||
|
||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||
{
|
||||
pageIds.Add(location.PageId);
|
||||
if (TryReadFirstOverflowPage(location, out var firstOverflowPage))
|
||||
{
|
||||
AddOverflowChainPages(pageIds, firstOverflowPage);
|
||||
}
|
||||
}
|
||||
|
||||
int data = 0;
|
||||
int overflow = 0;
|
||||
int indexPages = 0;
|
||||
int other = 0;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
foreach (var pageId in pageIds)
|
||||
{
|
||||
if (pageId >= _pageFile.NextPageId)
|
||||
continue;
|
||||
|
||||
_pageFile.ReadPage(pageId, pageBuffer);
|
||||
var pageType = PageHeader.ReadFrom(pageBuffer).PageType;
|
||||
|
||||
if (pageType == PageType.Data)
|
||||
{
|
||||
data++;
|
||||
}
|
||||
else if (pageType == PageType.Overflow)
|
||||
{
|
||||
overflow++;
|
||||
}
|
||||
else if (pageType == PageType.Index || pageType == PageType.Vector || pageType == PageType.Spatial)
|
||||
{
|
||||
indexPages++;
|
||||
}
|
||||
else
|
||||
{
|
||||
other++;
|
||||
}
|
||||
}
|
||||
|
||||
results.Add(new CollectionPageUsageEntry
|
||||
{
|
||||
CollectionName = metadata.Name,
|
||||
TotalDistinctPages = pageIds.Count,
|
||||
DataPages = data,
|
||||
OverflowPages = overflow,
|
||||
IndexPages = indexPages,
|
||||
OtherPages = other
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets per-collection logical-vs-stored compression ratios.
|
||||
/// </summary>
|
||||
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
||||
{
|
||||
var metadataEntries = GetAllCollectionMetadata();
|
||||
var results = new List<CollectionCompressionRatioEntry>(metadataEntries.Count);
|
||||
|
||||
foreach (var metadata in metadataEntries)
|
||||
{
|
||||
long docs = 0;
|
||||
long compressedDocs = 0;
|
||||
long bytesBefore = 0;
|
||||
long bytesAfter = 0;
|
||||
|
||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||
{
|
||||
if (!TryReadSlotPayloadStats(location, out var isCompressed, out var originalBytes, out var storedBytes))
|
||||
continue;
|
||||
|
||||
docs++;
|
||||
if (isCompressed)
|
||||
compressedDocs++;
|
||||
|
||||
bytesBefore += originalBytes;
|
||||
bytesAfter += storedBytes;
|
||||
}
|
||||
|
||||
results.Add(new CollectionCompressionRatioEntry
|
||||
{
|
||||
CollectionName = metadata.Name,
|
||||
DocumentCount = docs,
|
||||
CompressedDocumentCount = compressedDocs,
|
||||
BytesBeforeCompression = bytesBefore,
|
||||
BytesAfterCompression = bytesAfter
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets free-list summary for diagnostics.
|
||||
/// </summary>
|
||||
public FreeListSummary GetFreeListSummary()
|
||||
{
|
||||
var snapshot = CaptureCompactionSnapshot();
|
||||
return new FreeListSummary
|
||||
{
|
||||
PageCount = snapshot.PageCount,
|
||||
FreePageCount = snapshot.FreePageCount,
|
||||
FreeBytes = snapshot.TotalFreeBytes,
|
||||
FragmentationPercent = snapshot.FragmentationPercent,
|
||||
TailReclaimablePages = snapshot.TailReclaimablePages
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets detailed page-level fragmentation diagnostics.
|
||||
/// </summary>
|
||||
public FragmentationMapReport GetFragmentationMap()
|
||||
{
|
||||
var freePageSet = new HashSet<uint>(_pageFile.EnumerateFreePages(includeEmptyPages: true));
|
||||
var pageCount = _pageFile.NextPageId;
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var pages = new List<FragmentationPageEntry>((int)pageCount);
|
||||
|
||||
long totalFreeBytes = 0;
|
||||
|
||||
for (uint pageId = 0; pageId < pageCount; pageId++)
|
||||
{
|
||||
_pageFile.ReadPage(pageId, buffer);
|
||||
var pageHeader = PageHeader.ReadFrom(buffer);
|
||||
var isFreePage = freePageSet.Contains(pageId);
|
||||
|
||||
int freeBytes = 0;
|
||||
if (isFreePage)
|
||||
{
|
||||
freeBytes = _pageFile.PageSize;
|
||||
}
|
||||
else if (TryReadSlottedFreeSpace(buffer, out var slottedFreeBytes))
|
||||
{
|
||||
freeBytes = slottedFreeBytes;
|
||||
}
|
||||
|
||||
totalFreeBytes += freeBytes;
|
||||
|
||||
pages.Add(new FragmentationPageEntry
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = pageHeader.PageType,
|
||||
IsFreePage = isFreePage,
|
||||
FreeBytes = freeBytes
|
||||
});
|
||||
}
|
||||
|
||||
uint tailReclaimablePages = 0;
|
||||
for (var i = pageCount; i > 2; i--)
|
||||
{
|
||||
if (!freePageSet.Contains(i - 1))
|
||||
break;
|
||||
|
||||
tailReclaimablePages++;
|
||||
}
|
||||
|
||||
var fileBytes = Math.Max(1L, _pageFile.FileLengthBytes);
|
||||
return new FragmentationMapReport
|
||||
{
|
||||
Pages = pages,
|
||||
TotalFreeBytes = totalFreeBytes,
|
||||
FragmentationPercent = (totalFreeBytes * 100d) / fileBytes,
|
||||
TailReclaimablePages = tailReclaimablePages
|
||||
};
|
||||
}
|
||||
|
||||
private IEnumerable<DocumentLocation> EnumeratePrimaryLocations(CollectionMetadata metadata)
|
||||
{
|
||||
if (metadata.PrimaryRootPageId == 0)
|
||||
yield break;
|
||||
|
||||
var index = new BTreeIndex(this, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
||||
|
||||
foreach (var entry in index.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transactionId: 0))
|
||||
{
|
||||
yield return entry.Location;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadFirstOverflowPage(in DocumentLocation location, out uint firstOverflowPage)
|
||||
{
|
||||
firstOverflowPage = 0;
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= header.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
if ((slot.Flags & SlotFlags.HasOverflow) == 0)
|
||||
return false;
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
firstOverflowPage = BinaryPrimitives.ReadUInt32LittleEndian(pageBuffer.AsSpan(slot.Offset + 4, 4));
|
||||
return true;
|
||||
}
|
||||
|
||||
private void AddOverflowChainPages(HashSet<uint> pageIds, uint firstOverflowPage)
|
||||
{
|
||||
if (firstOverflowPage == 0)
|
||||
return;
|
||||
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var visited = new HashSet<uint>();
|
||||
var current = firstOverflowPage;
|
||||
|
||||
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
||||
{
|
||||
pageIds.Add(current);
|
||||
_pageFile.ReadPage(current, buffer);
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
if (header.PageType != PageType.Overflow)
|
||||
break;
|
||||
|
||||
current = header.NextOverflowPage;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadSlotPayloadStats(
|
||||
in DocumentLocation location,
|
||||
out bool isCompressed,
|
||||
out int originalBytes,
|
||||
out int storedBytes)
|
||||
{
|
||||
isCompressed = false;
|
||||
originalBytes = 0;
|
||||
storedBytes = 0;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= header.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
var hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
||||
|
||||
if (!hasOverflow)
|
||||
{
|
||||
storedBytes = slot.Length;
|
||||
if (!isCompressed)
|
||||
{
|
||||
originalBytes = slot.Length;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < CompressedPayloadHeader.Size)
|
||||
return false;
|
||||
|
||||
var compressedHeader = CompressedPayloadHeader.ReadFrom(pageBuffer.AsSpan(slot.Offset, CompressedPayloadHeader.Size));
|
||||
originalBytes = compressedHeader.OriginalLength;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||
var totalStoredBytes = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
||||
if (totalStoredBytes < 0)
|
||||
return false;
|
||||
|
||||
storedBytes = totalStoredBytes;
|
||||
if (!isCompressed)
|
||||
{
|
||||
originalBytes = totalStoredBytes;
|
||||
return true;
|
||||
}
|
||||
|
||||
var storedPrefix = primaryPayload.Slice(8);
|
||||
Span<byte> headerBuffer = stackalloc byte[CompressedPayloadHeader.Size];
|
||||
|
||||
if (storedPrefix.Length >= CompressedPayloadHeader.Size)
|
||||
{
|
||||
storedPrefix.Slice(0, CompressedPayloadHeader.Size).CopyTo(headerBuffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
storedPrefix.CopyTo(headerBuffer);
|
||||
var copied = storedPrefix.Length;
|
||||
var nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
||||
var overflowBuffer = new byte[_pageFile.PageSize];
|
||||
|
||||
while (copied < CompressedPayloadHeader.Size && nextOverflow != 0 && nextOverflow < _pageFile.NextPageId)
|
||||
{
|
||||
_pageFile.ReadPage(nextOverflow, overflowBuffer);
|
||||
var overflowHeader = SlottedPageHeader.ReadFrom(overflowBuffer);
|
||||
if (overflowHeader.PageType != PageType.Overflow)
|
||||
return false;
|
||||
|
||||
var available = Math.Min(CompressedPayloadHeader.Size - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
||||
overflowBuffer.AsSpan(SlottedPageHeader.Size, available).CopyTo(headerBuffer.Slice(copied));
|
||||
copied += available;
|
||||
nextOverflow = overflowHeader.NextOverflowPage;
|
||||
}
|
||||
|
||||
if (copied < CompressedPayloadHeader.Size)
|
||||
return false;
|
||||
}
|
||||
|
||||
var headerFromPayload = CompressedPayloadHeader.ReadFrom(headerBuffer);
|
||||
originalBytes = headerFromPayload.OriginalLength;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
219
src/CBDD.Core/Storage/StorageEngine.Dictionary.cs
Executable file
219
src/CBDD.Core/Storage/StorageEngine.Dictionary.cs
Executable file
@@ -0,0 +1,219 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, ushort> _dictionaryCache = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<ushort, string> _dictionaryReverseCache = new();
|
||||
private uint _dictionaryRootPageId;
|
||||
private ushort _nextDictionaryId;
|
||||
|
||||
// Lock for dictionary modifications (simple lock for now, could be RW lock)
|
||||
private readonly object _dictionaryLock = new();
|
||||
|
||||
private void InitializeDictionary()
|
||||
{
|
||||
// 1. Read File Header (Page 0) to get Dictionary Root
|
||||
var headerBuffer = new byte[PageSize];
|
||||
ReadPage(0, null, headerBuffer);
|
||||
var header = PageHeader.ReadFrom(headerBuffer);
|
||||
|
||||
if (header.DictionaryRootPageId == 0)
|
||||
{
|
||||
// Initialize new Dictionary
|
||||
lock (_dictionaryLock)
|
||||
{
|
||||
// Double check
|
||||
ReadPage(0, null, headerBuffer);
|
||||
header = PageHeader.ReadFrom(headerBuffer);
|
||||
if (header.DictionaryRootPageId == 0)
|
||||
{
|
||||
_dictionaryRootPageId = AllocatePage();
|
||||
|
||||
// Init Dictionary Page
|
||||
var pageBuffer = new byte[PageSize];
|
||||
DictionaryPage.Initialize(pageBuffer, _dictionaryRootPageId);
|
||||
WritePageImmediate(_dictionaryRootPageId, pageBuffer);
|
||||
|
||||
// Update Header
|
||||
header.DictionaryRootPageId = _dictionaryRootPageId;
|
||||
header.WriteTo(headerBuffer);
|
||||
WritePageImmediate(0, headerBuffer);
|
||||
|
||||
// Init Next ID
|
||||
_nextDictionaryId = DictionaryPage.ReservedValuesEnd + 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
_dictionaryRootPageId = header.DictionaryRootPageId;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_dictionaryRootPageId = header.DictionaryRootPageId;
|
||||
|
||||
// Warm cache
|
||||
ushort maxId = DictionaryPage.ReservedValuesEnd;
|
||||
foreach (var (key, val) in DictionaryPage.FindAllGlobal(this, _dictionaryRootPageId))
|
||||
{
|
||||
var lowerKey = key.ToLowerInvariant();
|
||||
_dictionaryCache[lowerKey] = val;
|
||||
_dictionaryReverseCache[val] = lowerKey;
|
||||
if (val > maxId) maxId = val;
|
||||
}
|
||||
_nextDictionaryId = (ushort)(maxId + 1);
|
||||
}
|
||||
|
||||
// Pre-register internal keys used for Schema persistence
|
||||
RegisterKeys(new[] { "_id", "t", "_v", "f", "n", "b", "s", "a" });
|
||||
|
||||
// Pre-register common array indices to avoid mapping during high-frequency writes
|
||||
var indices = new List<string>(101);
|
||||
for (int i = 0; i <= 100; i++) indices.Add(i.ToString());
|
||||
RegisterKeys(indices);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the key-to-id dictionary cache.
|
||||
/// </summary>
|
||||
/// <returns>The key-to-id map.</returns>
|
||||
public ConcurrentDictionary<string, ushort> GetKeyMap() => _dictionaryCache;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the id-to-key dictionary cache.
|
||||
/// </summary>
|
||||
/// <returns>The id-to-key map.</returns>
|
||||
public ConcurrentDictionary<ushort, string> GetKeyReverseMap() => _dictionaryReverseCache;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the ID for a dictionary key, creating it if it doesn't exist.
|
||||
/// Thread-safe.
|
||||
/// </summary>
|
||||
/// <param name="key">The dictionary key.</param>
|
||||
/// <returns>The dictionary identifier for the key.</returns>
|
||||
public ushort GetOrAddDictionaryEntry(string key)
|
||||
{
|
||||
key = key.ToLowerInvariant();
|
||||
if (_dictionaryCache.TryGetValue(key, out var id))
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
lock (_dictionaryLock)
|
||||
{
|
||||
// Double checked locking
|
||||
if (_dictionaryCache.TryGetValue(key, out id))
|
||||
{
|
||||
return id;
|
||||
}
|
||||
|
||||
// Try to find in storage (in case cache is incomplete or another process?)
|
||||
// Note: FindAllGlobal loaded everything, so strict cache miss means it's not in DB.
|
||||
// BUT if we support concurrent writers (multiple processed), we should re-check DB.
|
||||
// Current CBDD seems to be single-process exclusive lock (FileShare.None).
|
||||
// So in-memory cache is authoritative after load.
|
||||
|
||||
// Generate New ID
|
||||
ushort nextId = _nextDictionaryId;
|
||||
if (nextId == 0) nextId = DictionaryPage.ReservedValuesEnd + 1; // Should be init, but safety
|
||||
|
||||
// Insert into Page
|
||||
// usage of default(ulong) or null transaction?
|
||||
// Dictionary updates should ideally be transactional or immediate?
|
||||
// "Immediate" for now to simplify, as dictionary is cross-collection.
|
||||
// If we use transaction, we need to pass it in. For now, immediate write.
|
||||
|
||||
// We need to support "Insert Global" which handles overflow.
|
||||
// DictionaryPage.Insert only handles single page.
|
||||
|
||||
// We need logic here to traverse chain and find space.
|
||||
if (InsertDictionaryEntryGlobal(key, nextId))
|
||||
{
|
||||
_dictionaryCache[key] = nextId;
|
||||
_dictionaryReverseCache[nextId] = key;
|
||||
_nextDictionaryId++;
|
||||
return nextId;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new InvalidOperationException("Failed to insert dictionary entry (Storage Full?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the dictionary key for an identifier.
|
||||
/// </summary>
|
||||
/// <param name="id">The dictionary identifier.</param>
|
||||
/// <returns>The dictionary key if found; otherwise, <see langword="null"/>.</returns>
|
||||
public string? GetDictionaryKey(ushort id)
|
||||
{
|
||||
if (_dictionaryReverseCache.TryGetValue(id, out var key))
|
||||
return key;
|
||||
return null;
|
||||
}
|
||||
|
||||
private bool InsertDictionaryEntryGlobal(string key, ushort value)
|
||||
{
|
||||
var pageId = _dictionaryRootPageId;
|
||||
var pageBuffer = new byte[PageSize];
|
||||
|
||||
while (true)
|
||||
{
|
||||
ReadPage(pageId, null, pageBuffer);
|
||||
|
||||
// Try Insert
|
||||
if (DictionaryPage.Insert(pageBuffer, key, value))
|
||||
{
|
||||
// Success - Write Back
|
||||
WritePageImmediate(pageId, pageBuffer);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Page Full - Check Next Page
|
||||
var header = PageHeader.ReadFrom(pageBuffer);
|
||||
if (header.NextPageId != 0)
|
||||
{
|
||||
pageId = header.NextPageId;
|
||||
continue;
|
||||
}
|
||||
|
||||
// No Next Page - Allocate New
|
||||
var newPageId = AllocatePage();
|
||||
var newPageBuffer = new byte[PageSize];
|
||||
DictionaryPage.Initialize(newPageBuffer, newPageId);
|
||||
|
||||
// Should likely insert into NEW page immediately to save I/O?
|
||||
// Or just link and loop?
|
||||
// Let's Insert into new page logic here to avoid re-reading.
|
||||
if (!DictionaryPage.Insert(newPageBuffer, key, value))
|
||||
return false; // Should not happen on empty page unless key is huge > page
|
||||
|
||||
// Write New Page
|
||||
WritePageImmediate(newPageId, newPageBuffer);
|
||||
|
||||
// Update Previous Page Link
|
||||
header.NextPageId = newPageId;
|
||||
header.WriteTo(pageBuffer);
|
||||
WritePageImmediate(pageId, pageBuffer);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a set of keys in the global dictionary.
|
||||
/// Ensures all keys are assigned an ID and persisted.
|
||||
/// </summary>
|
||||
/// <param name="keys">The keys to register.</param>
|
||||
public void RegisterKeys(IEnumerable<string> keys)
|
||||
{
|
||||
foreach (var key in keys)
|
||||
{
|
||||
GetOrAddDictionaryEntry(key.ToLowerInvariant());
|
||||
}
|
||||
}
|
||||
}
|
||||
103
src/CBDD.Core/Storage/StorageEngine.Format.cs
Normal file
103
src/CBDD.Core/Storage/StorageEngine.Format.cs
Normal file
@@ -0,0 +1,103 @@
|
||||
using System.Buffers.Binary;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
[Flags]
|
||||
internal enum StorageFeatureFlags : byte
|
||||
{
|
||||
None = 0,
|
||||
CompressionCapability = 1 << 0
|
||||
}
|
||||
|
||||
internal readonly struct StorageFormatMetadata
|
||||
{
|
||||
internal const int WireSize = 16;
|
||||
|
||||
public bool IsPresent { get; }
|
||||
public byte Version { get; }
|
||||
public StorageFeatureFlags FeatureFlags { get; }
|
||||
public CompressionCodec DefaultCodec { get; }
|
||||
|
||||
public bool CompressionCapabilityEnabled => (FeatureFlags & StorageFeatureFlags.CompressionCapability) != 0;
|
||||
|
||||
private StorageFormatMetadata(bool isPresent, byte version, StorageFeatureFlags featureFlags, CompressionCodec defaultCodec)
|
||||
{
|
||||
IsPresent = isPresent;
|
||||
Version = version;
|
||||
FeatureFlags = featureFlags;
|
||||
DefaultCodec = defaultCodec;
|
||||
}
|
||||
|
||||
public static StorageFormatMetadata Present(byte version, StorageFeatureFlags featureFlags, CompressionCodec defaultCodec)
|
||||
{
|
||||
return new StorageFormatMetadata(true, version, featureFlags, defaultCodec);
|
||||
}
|
||||
|
||||
public static StorageFormatMetadata Legacy(CompressionCodec defaultCodec)
|
||||
{
|
||||
return new StorageFormatMetadata(false, 0, StorageFeatureFlags.None, defaultCodec);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class StorageFormatConstants
|
||||
{
|
||||
internal const ushort SlottedPageFormatMarker = 0x4301;
|
||||
}
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
private const int StorageHeaderExtensionOffset = 0;
|
||||
private const uint StorageFormatMagic = 0x4D464243u; // "CBFM" little-endian
|
||||
private const byte CurrentStorageFormatVersion = 1;
|
||||
|
||||
private StorageFormatMetadata InitializeStorageFormatMetadata()
|
||||
{
|
||||
Span<byte> metadataBuffer = stackalloc byte[StorageFormatMetadata.WireSize];
|
||||
_pageFile.ReadPageZeroExtension(StorageHeaderExtensionOffset, metadataBuffer);
|
||||
|
||||
if (TryReadStorageFormatMetadata(metadataBuffer, out var metadata))
|
||||
return metadata;
|
||||
|
||||
if (!_pageFile.WasCreated)
|
||||
return StorageFormatMetadata.Legacy(_compressionOptions.Codec);
|
||||
|
||||
var featureFlags = _compressionOptions.EnableCompression
|
||||
? StorageFeatureFlags.CompressionCapability
|
||||
: StorageFeatureFlags.None;
|
||||
var initialMetadata = StorageFormatMetadata.Present(CurrentStorageFormatVersion, featureFlags, _compressionOptions.Codec);
|
||||
WriteStorageFormatMetadata(initialMetadata);
|
||||
return initialMetadata;
|
||||
}
|
||||
|
||||
private static bool TryReadStorageFormatMetadata(ReadOnlySpan<byte> source, out StorageFormatMetadata metadata)
|
||||
{
|
||||
metadata = default;
|
||||
if (source.Length < StorageFormatMetadata.WireSize)
|
||||
return false;
|
||||
|
||||
var magic = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(0, 4));
|
||||
if (magic != StorageFormatMagic)
|
||||
return false;
|
||||
|
||||
var version = source[4];
|
||||
var featureFlags = (StorageFeatureFlags)source[5];
|
||||
var codec = (CompressionCodec)source[6];
|
||||
if (!Enum.IsDefined(codec))
|
||||
codec = CompressionCodec.None;
|
||||
|
||||
metadata = StorageFormatMetadata.Present(version, featureFlags, codec);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void WriteStorageFormatMetadata(StorageFormatMetadata metadata)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[StorageFormatMetadata.WireSize];
|
||||
buffer.Clear();
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(buffer.Slice(0, 4), StorageFormatMagic);
|
||||
buffer[4] = metadata.Version;
|
||||
buffer[5] = (byte)metadata.FeatureFlags;
|
||||
buffer[6] = (byte)metadata.DefaultCodec;
|
||||
_pageFile.WritePageZeroExtension(StorageHeaderExtensionOffset, buffer);
|
||||
}
|
||||
}
|
||||
1210
src/CBDD.Core/Storage/StorageEngine.Maintenance.cs
Normal file
1210
src/CBDD.Core/Storage/StorageEngine.Maintenance.cs
Normal file
File diff suppressed because it is too large
Load Diff
24
src/CBDD.Core/Storage/StorageEngine.Memory.cs
Executable file
24
src/CBDD.Core/Storage/StorageEngine.Memory.cs
Executable file
@@ -0,0 +1,24 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Allocates a new page.
|
||||
/// </summary>
|
||||
/// <returns>Page ID of the allocated page</returns>
|
||||
public uint AllocatePage()
|
||||
{
|
||||
return _pageFile.AllocatePage();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Frees a page.
|
||||
/// </summary>
|
||||
/// <param name="pageId">Page to free</param>
|
||||
public void FreePage(uint pageId)
|
||||
{
|
||||
_pageFile.FreePage(pageId);
|
||||
}
|
||||
}
|
||||
496
src/CBDD.Core/Storage/StorageEngine.Migration.cs
Normal file
496
src/CBDD.Core/Storage/StorageEngine.Migration.cs
Normal file
@@ -0,0 +1,496 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Options controlling compression migration.
|
||||
/// </summary>
|
||||
public sealed class CompressionMigrationOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Enables dry-run estimation without mutating database contents.
|
||||
/// </summary>
|
||||
public bool DryRun { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Target codec for migrated payloads.
|
||||
/// </summary>
|
||||
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
||||
|
||||
/// <summary>
|
||||
/// Target compression level.
|
||||
/// </summary>
|
||||
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum logical payload size required before compression is attempted.
|
||||
/// </summary>
|
||||
public int MinSizeBytes { get; init; } = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum savings percent required to keep compressed output.
|
||||
/// </summary>
|
||||
public int MinSavingsPercent { get; init; } = 10;
|
||||
|
||||
/// <summary>
|
||||
/// Optional include-only collection list (case-insensitive).
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? IncludeCollections { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional exclusion collection list (case-insensitive).
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? ExcludeCollections { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a compression migration run.
|
||||
/// </summary>
|
||||
public sealed class CompressionMigrationResult
|
||||
{
|
||||
public bool DryRun { get; init; }
|
||||
public CompressionCodec Codec { get; init; }
|
||||
public CompressionLevel Level { get; init; }
|
||||
public int CollectionsProcessed { get; init; }
|
||||
public long DocumentsScanned { get; init; }
|
||||
public long DocumentsRewritten { get; init; }
|
||||
public long DocumentsSkipped { get; init; }
|
||||
public long BytesBefore { get; init; }
|
||||
public long BytesEstimatedAfter { get; init; }
|
||||
public long BytesActualAfter { get; init; }
|
||||
}
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Estimates or applies a one-time compression migration.
|
||||
/// </summary>
|
||||
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
||||
{
|
||||
return MigrateCompressionAsync(options).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Estimates or applies a one-time compression migration.
|
||||
/// </summary>
|
||||
public async Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null, CancellationToken ct = default)
|
||||
{
|
||||
var normalized = NormalizeMigrationOptions(options);
|
||||
|
||||
await _maintenanceGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
EnsureNoActiveTransactions();
|
||||
CheckpointInternal();
|
||||
|
||||
var collections = ResolveMigrationCollections(normalized);
|
||||
|
||||
long docsScanned = 0;
|
||||
long docsRewritten = 0;
|
||||
long docsSkipped = 0;
|
||||
long bytesBefore = 0;
|
||||
long bytesEstimatedAfter = 0;
|
||||
long bytesActualAfter = 0;
|
||||
|
||||
foreach (var metadata in collections)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
if (!TryReadStoredPayload(location, out var storedPayload, out var isCompressed))
|
||||
{
|
||||
docsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryGetLogicalPayload(storedPayload, isCompressed, out var logicalPayload))
|
||||
{
|
||||
docsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
docsScanned++;
|
||||
bytesBefore += logicalPayload.Length;
|
||||
|
||||
var targetStored = BuildTargetStoredPayload(logicalPayload, normalized, out var targetCompressed);
|
||||
bytesEstimatedAfter += targetStored.Length;
|
||||
|
||||
if (normalized.DryRun)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryRewriteStoredPayloadAtLocation(location, targetStored, targetCompressed, out var actualStoredBytes))
|
||||
{
|
||||
docsSkipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
docsRewritten++;
|
||||
bytesActualAfter += actualStoredBytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (!normalized.DryRun)
|
||||
{
|
||||
var metadata = StorageFormatMetadata.Present(
|
||||
version: 1,
|
||||
featureFlags: StorageFeatureFlags.CompressionCapability,
|
||||
defaultCodec: normalized.Codec);
|
||||
WriteStorageFormatMetadata(metadata);
|
||||
_pageFile.Flush();
|
||||
}
|
||||
|
||||
return new CompressionMigrationResult
|
||||
{
|
||||
DryRun = normalized.DryRun,
|
||||
Codec = normalized.Codec,
|
||||
Level = normalized.Level,
|
||||
CollectionsProcessed = collections.Count,
|
||||
DocumentsScanned = docsScanned,
|
||||
DocumentsRewritten = docsRewritten,
|
||||
DocumentsSkipped = docsSkipped,
|
||||
BytesBefore = bytesBefore,
|
||||
BytesEstimatedAfter = bytesEstimatedAfter,
|
||||
BytesActualAfter = normalized.DryRun ? 0 : bytesActualAfter
|
||||
};
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_maintenanceGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static CompressionMigrationOptions NormalizeMigrationOptions(CompressionMigrationOptions? options)
|
||||
{
|
||||
var normalized = options ?? new CompressionMigrationOptions();
|
||||
|
||||
if (!Enum.IsDefined(normalized.Codec) || normalized.Codec == CompressionCodec.None)
|
||||
throw new ArgumentOutOfRangeException(nameof(options), "Migration codec must be a supported non-None codec.");
|
||||
|
||||
if (normalized.MinSizeBytes < 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(options), "MinSizeBytes must be non-negative.");
|
||||
|
||||
if (normalized.MinSavingsPercent is < 0 or > 100)
|
||||
throw new ArgumentOutOfRangeException(nameof(options), "MinSavingsPercent must be between 0 and 100.");
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
private List<CollectionMetadata> ResolveMigrationCollections(CompressionMigrationOptions options)
|
||||
{
|
||||
var collections = GetAllCollectionMetadata();
|
||||
|
||||
var includes = options.IncludeCollections is { Count: > 0 }
|
||||
? new HashSet<string>(options.IncludeCollections, StringComparer.OrdinalIgnoreCase)
|
||||
: null;
|
||||
|
||||
var excludes = options.ExcludeCollections is { Count: > 0 }
|
||||
? new HashSet<string>(options.ExcludeCollections, StringComparer.OrdinalIgnoreCase)
|
||||
: null;
|
||||
|
||||
return collections
|
||||
.Where(c => includes == null || includes.Contains(c.Name))
|
||||
.Where(c => excludes == null || !excludes.Contains(c.Name))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private byte[] BuildTargetStoredPayload(ReadOnlySpan<byte> logicalPayload, CompressionMigrationOptions options, out bool compressed)
|
||||
{
|
||||
compressed = false;
|
||||
|
||||
if (logicalPayload.Length < options.MinSizeBytes)
|
||||
return logicalPayload.ToArray();
|
||||
|
||||
try
|
||||
{
|
||||
var compressedPayload = _compressionService.Compress(logicalPayload, options.Codec, options.Level);
|
||||
var storedLength = CompressedPayloadHeader.Size + compressedPayload.Length;
|
||||
var savings = logicalPayload.Length - storedLength;
|
||||
var savingsPercent = logicalPayload.Length == 0 ? 0 : (int)((savings * 100L) / logicalPayload.Length);
|
||||
if (savings <= 0 || savingsPercent < options.MinSavingsPercent)
|
||||
return logicalPayload.ToArray();
|
||||
|
||||
var output = new byte[storedLength];
|
||||
var header = CompressedPayloadHeader.Create(options.Codec, logicalPayload.Length, compressedPayload);
|
||||
header.WriteTo(output.AsSpan(0, CompressedPayloadHeader.Size));
|
||||
compressedPayload.CopyTo(output.AsSpan(CompressedPayloadHeader.Size));
|
||||
compressed = true;
|
||||
return output;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return logicalPayload.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryGetLogicalPayload(ReadOnlySpan<byte> storedPayload, bool isCompressed, out byte[] logicalPayload)
|
||||
{
|
||||
logicalPayload = Array.Empty<byte>();
|
||||
|
||||
if (!isCompressed)
|
||||
{
|
||||
logicalPayload = storedPayload.ToArray();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (storedPayload.Length < CompressedPayloadHeader.Size)
|
||||
return false;
|
||||
|
||||
var header = CompressedPayloadHeader.ReadFrom(storedPayload.Slice(0, CompressedPayloadHeader.Size));
|
||||
if (!Enum.IsDefined(header.Codec) || header.Codec == CompressionCodec.None)
|
||||
return false;
|
||||
|
||||
if (header.OriginalLength < 0 || header.CompressedLength < 0)
|
||||
return false;
|
||||
|
||||
if (CompressedPayloadHeader.Size + header.CompressedLength > storedPayload.Length)
|
||||
return false;
|
||||
|
||||
var compressedPayload = storedPayload.Slice(CompressedPayloadHeader.Size, header.CompressedLength);
|
||||
if (!header.ValidateChecksum(compressedPayload))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
logicalPayload = _compressionService.Decompress(
|
||||
compressedPayload,
|
||||
header.Codec,
|
||||
header.OriginalLength,
|
||||
Math.Max(header.OriginalLength, _compressionOptions.MaxDecompressedSizeBytes));
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private bool TryReadStoredPayload(in DocumentLocation location, out byte[] storedPayload, out bool isCompressed)
|
||||
{
|
||||
storedPayload = Array.Empty<byte>();
|
||||
isCompressed = false;
|
||||
|
||||
if (location.PageId >= _pageFile.NextPageId)
|
||||
return false;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var header = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= header.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
||||
var hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||
|
||||
if (!hasOverflow)
|
||||
{
|
||||
storedPayload = pageBuffer.AsSpan(slot.Offset, slot.Length).ToArray();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||
var totalStoredLength = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
||||
var nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
||||
if (totalStoredLength < 0)
|
||||
return false;
|
||||
|
||||
var output = new byte[totalStoredLength];
|
||||
var primaryChunk = primaryPayload.Slice(8);
|
||||
var copied = Math.Min(primaryChunk.Length, output.Length);
|
||||
primaryChunk.Slice(0, copied).CopyTo(output);
|
||||
|
||||
var overflowBuffer = new byte[_pageFile.PageSize];
|
||||
while (copied < output.Length && nextOverflow != 0 && nextOverflow < _pageFile.NextPageId)
|
||||
{
|
||||
_pageFile.ReadPage(nextOverflow, overflowBuffer);
|
||||
var overflowHeader = SlottedPageHeader.ReadFrom(overflowBuffer);
|
||||
if (overflowHeader.PageType != PageType.Overflow)
|
||||
return false;
|
||||
|
||||
var chunk = Math.Min(output.Length - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
||||
overflowBuffer.AsSpan(SlottedPageHeader.Size, chunk).CopyTo(output.AsSpan(copied));
|
||||
copied += chunk;
|
||||
nextOverflow = overflowHeader.NextOverflowPage;
|
||||
}
|
||||
|
||||
if (copied != output.Length)
|
||||
return false;
|
||||
|
||||
storedPayload = output;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryRewriteStoredPayloadAtLocation(
|
||||
in DocumentLocation location,
|
||||
ReadOnlySpan<byte> newStoredPayload,
|
||||
bool compressed,
|
||||
out int actualStoredBytes)
|
||||
{
|
||||
actualStoredBytes = 0;
|
||||
|
||||
if (location.PageId >= _pageFile.NextPageId)
|
||||
return false;
|
||||
|
||||
var pageBuffer = new byte[_pageFile.PageSize];
|
||||
_pageFile.ReadPage(location.PageId, pageBuffer);
|
||||
|
||||
var pageHeader = SlottedPageHeader.ReadFrom(pageBuffer);
|
||||
if (location.SlotIndex >= pageHeader.SlotCount)
|
||||
return false;
|
||||
|
||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||
return false;
|
||||
|
||||
var oldHasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||
uint oldOverflowHead = 0;
|
||||
if (oldHasOverflow)
|
||||
{
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
oldOverflowHead = BinaryPrimitives.ReadUInt32LittleEndian(pageBuffer.AsSpan(slot.Offset + 4, 4));
|
||||
}
|
||||
|
||||
if (newStoredPayload.Length <= slot.Length)
|
||||
{
|
||||
var destination = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||
destination.Clear();
|
||||
newStoredPayload.CopyTo(destination);
|
||||
|
||||
slot.Length = (ushort)newStoredPayload.Length;
|
||||
slot.Flags &= ~(SlotFlags.Compressed | SlotFlags.HasOverflow);
|
||||
if (compressed)
|
||||
slot.Flags |= SlotFlags.Compressed;
|
||||
|
||||
slot.WriteTo(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
_pageFile.WritePage(location.PageId, pageBuffer);
|
||||
|
||||
if (oldOverflowHead != 0)
|
||||
FreeOverflowChainPages(oldOverflowHead);
|
||||
|
||||
actualStoredBytes = newStoredPayload.Length;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (slot.Length < 8)
|
||||
return false;
|
||||
|
||||
var primaryChunkSize = slot.Length - 8;
|
||||
if (primaryChunkSize < 0)
|
||||
return false;
|
||||
|
||||
var remainder = newStoredPayload.Slice(primaryChunkSize);
|
||||
var newOverflowHead = BuildOverflowChainForMigration(remainder);
|
||||
|
||||
var slotPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||
slotPayload.Clear();
|
||||
BinaryPrimitives.WriteInt32LittleEndian(slotPayload.Slice(0, 4), newStoredPayload.Length);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(slotPayload.Slice(4, 4), newOverflowHead);
|
||||
newStoredPayload.Slice(0, primaryChunkSize).CopyTo(slotPayload.Slice(8));
|
||||
|
||||
slot.Flags &= ~(SlotFlags.Compressed | SlotFlags.HasOverflow);
|
||||
slot.Flags |= SlotFlags.HasOverflow;
|
||||
if (compressed)
|
||||
slot.Flags |= SlotFlags.Compressed;
|
||||
|
||||
slot.WriteTo(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||
_pageFile.WritePage(location.PageId, pageBuffer);
|
||||
|
||||
if (oldOverflowHead != 0)
|
||||
FreeOverflowChainPages(oldOverflowHead);
|
||||
|
||||
actualStoredBytes = newStoredPayload.Length;
|
||||
return true;
|
||||
}
|
||||
|
||||
private uint BuildOverflowChainForMigration(ReadOnlySpan<byte> overflowPayload)
|
||||
{
|
||||
if (overflowPayload.IsEmpty)
|
||||
return 0;
|
||||
|
||||
var chunkSize = _pageFile.PageSize - SlottedPageHeader.Size;
|
||||
uint nextOverflowPageId = 0;
|
||||
|
||||
var tailSize = overflowPayload.Length % chunkSize;
|
||||
var fullPages = overflowPayload.Length / chunkSize;
|
||||
|
||||
if (tailSize > 0)
|
||||
{
|
||||
var tailOffset = fullPages * chunkSize;
|
||||
var tailSlice = overflowPayload.Slice(tailOffset, tailSize);
|
||||
nextOverflowPageId = AllocateOverflowPageForMigration(tailSlice, nextOverflowPageId);
|
||||
}
|
||||
|
||||
for (var i = fullPages - 1; i >= 0; i--)
|
||||
{
|
||||
var chunkOffset = i * chunkSize;
|
||||
var chunk = overflowPayload.Slice(chunkOffset, chunkSize);
|
||||
nextOverflowPageId = AllocateOverflowPageForMigration(chunk, nextOverflowPageId);
|
||||
}
|
||||
|
||||
return nextOverflowPageId;
|
||||
}
|
||||
|
||||
private uint AllocateOverflowPageForMigration(ReadOnlySpan<byte> payloadChunk, uint nextOverflowPageId)
|
||||
{
|
||||
var pageId = _pageFile.AllocatePage();
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
|
||||
var header = new SlottedPageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Overflow,
|
||||
SlotCount = 0,
|
||||
FreeSpaceStart = SlottedPageHeader.Size,
|
||||
FreeSpaceEnd = (ushort)_pageFile.PageSize,
|
||||
NextOverflowPage = nextOverflowPageId,
|
||||
TransactionId = 0
|
||||
};
|
||||
|
||||
header.WriteTo(buffer);
|
||||
payloadChunk.CopyTo(buffer.AsSpan(SlottedPageHeader.Size));
|
||||
_pageFile.WritePage(pageId, buffer);
|
||||
return pageId;
|
||||
}
|
||||
|
||||
private void FreeOverflowChainPages(uint firstOverflowPage)
|
||||
{
|
||||
var buffer = new byte[_pageFile.PageSize];
|
||||
var visited = new HashSet<uint>();
|
||||
var current = firstOverflowPage;
|
||||
|
||||
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
||||
{
|
||||
_pageFile.ReadPage(current, buffer);
|
||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||
var next = header.NextOverflowPage;
|
||||
_pageFile.FreePage(current);
|
||||
current = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
80
src/CBDD.Core/Storage/StorageEngine.Pages.cs
Executable file
80
src/CBDD.Core/Storage/StorageEngine.Pages.cs
Executable file
@@ -0,0 +1,80 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads a page with transaction isolation.
|
||||
/// 1. Check WAL cache for uncommitted writes (Read Your Own Writes)
|
||||
/// 2. Check WAL index for committed writes (lazy replay)
|
||||
/// 3. Read from PageFile (committed baseline)
|
||||
/// </summary>
|
||||
/// <param name="pageId">Page to read</param>
|
||||
/// <param name="transactionId">Optional transaction ID for isolation</param>
|
||||
/// <param name="destination">Buffer to write page data</param>
|
||||
public void ReadPage(uint pageId, ulong? transactionId, Span<byte> destination)
|
||||
{
|
||||
// 1. Check transaction-local WAL cache (Read Your Own Writes)
|
||||
// transactionId=0 or null means "no active transaction, read committed only"
|
||||
if (transactionId.HasValue &&
|
||||
transactionId.Value != 0 &&
|
||||
_walCache.TryGetValue(transactionId.Value, out var txnPages) &&
|
||||
txnPages.TryGetValue(pageId, out var uncommittedData))
|
||||
{
|
||||
var length = Math.Min(uncommittedData.Length, destination.Length);
|
||||
uncommittedData.AsSpan(0, length).CopyTo(destination);
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Check WAL index (committed but not checkpointed)
|
||||
if (_walIndex.TryGetValue(pageId, out var committedData))
|
||||
{
|
||||
var length = Math.Min(committedData.Length, destination.Length);
|
||||
committedData.AsSpan(0, length).CopyTo(destination);
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Read committed baseline from PageFile
|
||||
_pageFile.ReadPage(pageId, destination);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a page within a transaction.
|
||||
/// Data goes to WAL cache immediately and becomes visible to that transaction only.
|
||||
/// Will be written to WAL on commit.
|
||||
/// </summary>
|
||||
/// <param name="pageId">Page to write</param>
|
||||
/// <param name="transactionId">Transaction ID owning this write</param>
|
||||
/// <param name="data">Page data</param>
|
||||
public void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (transactionId == 0)
|
||||
throw new InvalidOperationException("Cannot write without a transaction (transactionId=0 is reserved)");
|
||||
|
||||
// Get or create transaction-local cache
|
||||
var txnPages = _walCache.GetOrAdd(transactionId,
|
||||
_ => new System.Collections.Concurrent.ConcurrentDictionary<uint, byte[]>());
|
||||
|
||||
// Store defensive copy
|
||||
var copy = data.ToArray();
|
||||
txnPages[pageId] = copy;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a page immediately to disk (non-transactional).
|
||||
/// Used for initialization and metadata updates outside of transactions.
|
||||
/// </summary>
|
||||
/// <param name="pageId">Page to write</param>
|
||||
/// <param name="data">Page data</param>
|
||||
public void WritePageImmediate(uint pageId, ReadOnlySpan<byte> data)
|
||||
{
|
||||
_pageFile.WritePage(pageId, data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of pages currently allocated in the page file.
|
||||
/// Useful for full database scans.
|
||||
/// </summary>
|
||||
public uint PageCount => _pageFile.NextPageId;
|
||||
}
|
||||
180
src/CBDD.Core/Storage/StorageEngine.Recovery.cs
Executable file
180
src/CBDD.Core/Storage/StorageEngine.Recovery.cs
Executable file
@@ -0,0 +1,180 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the current size of the WAL file.
|
||||
/// </summary>
|
||||
public long GetWalSize()
|
||||
{
|
||||
return _wal.GetCurrentSize();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Truncates the WAL file.
|
||||
/// Should only be called after a successful checkpoint.
|
||||
/// </summary>
|
||||
public void TruncateWal()
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flushes the WAL to disk.
|
||||
/// </summary>
|
||||
public void FlushWal()
|
||||
{
|
||||
_wal.Flush();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint: merges WAL into PageFile.
|
||||
/// Uses in-memory WAL index for efficiency and consistency.
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Performs a checkpoint: merges WAL into PageFile.
|
||||
/// Uses in-memory WAL index for efficiency and consistency.
|
||||
/// </summary>
|
||||
public void Checkpoint()
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
CheckpointInternal();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void CheckpointInternal()
|
||||
{
|
||||
if (_walIndex.IsEmpty)
|
||||
{
|
||||
// WAL may still contain begin/commit records for read-only transactions.
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Write all committed pages from index to PageFile
|
||||
foreach (var kvp in _walIndex)
|
||||
{
|
||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
// 2. Flush PageFile to ensure durability
|
||||
_pageFile.Flush();
|
||||
|
||||
// 3. Clear in-memory WAL index (now persisted)
|
||||
_walIndex.Clear();
|
||||
|
||||
// 4. Truncate WAL (all changes now in PageFile)
|
||||
_wal.Truncate();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a checkpoint asynchronously by merging WAL pages into the page file.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous checkpoint operation.</returns>
|
||||
public async Task CheckpointAsync(CancellationToken ct = default)
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
if (_walIndex.IsEmpty)
|
||||
{
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
_wal.Truncate();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Write all committed pages from index to PageFile
|
||||
// PageFile writes are sync (MMF), but that's fine as per plan (ValueTask strategy for MMF)
|
||||
foreach (var kvp in _walIndex)
|
||||
{
|
||||
_pageFile.WritePage(kvp.Key, kvp.Value);
|
||||
}
|
||||
|
||||
// 2. Flush PageFile to ensure durability
|
||||
_pageFile.Flush();
|
||||
|
||||
// 3. Clear in-memory WAL index (now persisted)
|
||||
_walIndex.Clear();
|
||||
|
||||
// 4. Truncate WAL (all changes now in PageFile)
|
||||
// WAL truncation involves file resize and flush
|
||||
// TODO: Add TruncateAsync to WAL? For now Truncate is sync.
|
||||
_wal.Truncate();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Recovers from crash by replaying WAL.
|
||||
/// Applies all committed transactions to PageFile, then truncates WAL.
|
||||
/// </summary>
|
||||
public void Recover()
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
// 1. Read WAL and identify committed transactions
|
||||
var records = _wal.ReadAll();
|
||||
var committedTxns = new HashSet<ulong>();
|
||||
var txnWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
||||
|
||||
foreach (var record in records)
|
||||
{
|
||||
if (record.Type == WalRecordType.Commit)
|
||||
committedTxns.Add(record.TransactionId);
|
||||
else if (record.Type == WalRecordType.Write)
|
||||
{
|
||||
if (!txnWrites.ContainsKey(record.TransactionId))
|
||||
txnWrites[record.TransactionId] = new List<(uint, byte[])>();
|
||||
|
||||
if (record.AfterImage != null)
|
||||
{
|
||||
txnWrites[record.TransactionId].Add((record.PageId, record.AfterImage));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Apply committed transactions to PageFile
|
||||
foreach (var txnId in committedTxns)
|
||||
{
|
||||
if (!txnWrites.ContainsKey(txnId))
|
||||
continue;
|
||||
|
||||
foreach (var (pageId, data) in txnWrites[txnId])
|
||||
{
|
||||
_pageFile.WritePage(pageId, data);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Flush PageFile to ensure durability
|
||||
_pageFile.Flush();
|
||||
|
||||
// 4. Clear in-memory WAL index (redundant since we just recovered)
|
||||
_walIndex.Clear();
|
||||
|
||||
// 5. Truncate WAL (all changes now in PageFile)
|
||||
_wal.Truncate();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
148
src/CBDD.Core/Storage/StorageEngine.Schema.cs
Executable file
148
src/CBDD.Core/Storage/StorageEngine.Schema.cs
Executable file
@@ -0,0 +1,148 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads all schemas from the schema page chain.
|
||||
/// </summary>
|
||||
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
||||
/// <returns>The list of schemas in chain order.</returns>
|
||||
public List<BsonSchema> GetSchemas(uint rootPageId)
|
||||
{
|
||||
var schemas = new List<BsonSchema>();
|
||||
if (rootPageId == 0) return schemas;
|
||||
|
||||
var pageId = rootPageId;
|
||||
var buffer = new byte[PageSize];
|
||||
|
||||
while (pageId != 0)
|
||||
{
|
||||
ReadPage(pageId, null, buffer);
|
||||
var header = PageHeader.ReadFrom(buffer);
|
||||
|
||||
if (header.PageType != PageType.Schema) break;
|
||||
|
||||
int used = PageSize - 32 - header.FreeBytes;
|
||||
if (used > 0)
|
||||
{
|
||||
var reader = new BsonSpanReader(buffer.AsSpan(32, used), GetKeyReverseMap());
|
||||
while (reader.Remaining >= 4)
|
||||
{
|
||||
var docSize = reader.PeekInt32();
|
||||
if (docSize <= 0 || docSize > reader.Remaining) break;
|
||||
|
||||
var schema = BsonSchema.FromBson(ref reader);
|
||||
schemas.Add(schema);
|
||||
}
|
||||
}
|
||||
|
||||
pageId = header.NextPageId;
|
||||
}
|
||||
|
||||
return schemas;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends a new schema version. Returns the root page ID (which might be new if it was 0 initially).
|
||||
/// </summary>
|
||||
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
||||
/// <param name="schema">The schema to append.</param>
|
||||
public uint AppendSchema(uint rootPageId, BsonSchema schema)
|
||||
{
|
||||
var buffer = new byte[PageSize];
|
||||
|
||||
// Serialize schema to temporary buffer to calculate size
|
||||
var tempBuffer = new byte[PageSize];
|
||||
var tempWriter = new BsonSpanWriter(tempBuffer, GetKeyMap());
|
||||
schema.ToBson(ref tempWriter);
|
||||
var schemaSize = tempWriter.Position;
|
||||
|
||||
if (rootPageId == 0)
|
||||
{
|
||||
rootPageId = AllocatePage();
|
||||
InitializeSchemaPage(buffer, rootPageId);
|
||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
||||
|
||||
var header = PageHeader.ReadFrom(buffer);
|
||||
header.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
||||
header.WriteTo(buffer);
|
||||
|
||||
WritePageImmediate(rootPageId, buffer);
|
||||
return rootPageId;
|
||||
}
|
||||
|
||||
// Find last page in chain
|
||||
uint currentPageId = rootPageId;
|
||||
uint lastPageId = rootPageId;
|
||||
while (currentPageId != 0)
|
||||
{
|
||||
ReadPage(currentPageId, null, buffer);
|
||||
var header = PageHeader.ReadFrom(buffer);
|
||||
lastPageId = currentPageId;
|
||||
if (header.NextPageId == 0) break;
|
||||
currentPageId = header.NextPageId;
|
||||
}
|
||||
|
||||
// Buffer now contains the last page
|
||||
var lastHeader = PageHeader.ReadFrom(buffer);
|
||||
int currentUsed = PageSize - 32 - lastHeader.FreeBytes;
|
||||
int lastOffset = 32 + currentUsed;
|
||||
|
||||
if (lastHeader.FreeBytes >= schemaSize)
|
||||
{
|
||||
// Fits in current page
|
||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(lastOffset));
|
||||
|
||||
lastHeader.FreeBytes -= (ushort)schemaSize;
|
||||
lastHeader.WriteTo(buffer);
|
||||
|
||||
WritePageImmediate(lastPageId, buffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Allocate new page
|
||||
var newPageId = AllocatePage();
|
||||
lastHeader.NextPageId = newPageId;
|
||||
lastHeader.WriteTo(buffer);
|
||||
WritePageImmediate(lastPageId, buffer);
|
||||
|
||||
InitializeSchemaPage(buffer, newPageId);
|
||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
||||
|
||||
var newHeader = PageHeader.ReadFrom(buffer);
|
||||
newHeader.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
||||
newHeader.WriteTo(buffer);
|
||||
|
||||
WritePageImmediate(newPageId, buffer);
|
||||
}
|
||||
|
||||
return rootPageId;
|
||||
}
|
||||
|
||||
private void InitializeSchemaPage(Span<byte> page, uint pageId)
|
||||
{
|
||||
var header = new PageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Schema,
|
||||
FreeBytes = (ushort)(page.Length - 32),
|
||||
NextPageId = 0,
|
||||
TransactionId = 0,
|
||||
Checksum = 0
|
||||
};
|
||||
page.Clear();
|
||||
header.WriteTo(page);
|
||||
}
|
||||
|
||||
private void AppendToSchemaPage(Span<byte> page, ref BsonSpanReader reader)
|
||||
{
|
||||
// reader contains the BSON doc
|
||||
var doc = reader.RemainingBytes();
|
||||
doc.CopyTo(page.Slice(32));
|
||||
}
|
||||
}
|
||||
356
src/CBDD.Core/Storage/StorageEngine.Transactions.cs
Executable file
356
src/CBDD.Core/Storage/StorageEngine.Transactions.cs
Executable file
@@ -0,0 +1,356 @@
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
public sealed partial class StorageEngine
|
||||
{
|
||||
#region Transaction Management
|
||||
|
||||
/// <summary>
|
||||
/// Begins a new transaction.
|
||||
/// </summary>
|
||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||
/// <returns>The started transaction.</returns>
|
||||
public Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
var txnId = _nextTransactionId++;
|
||||
var transaction = new Transaction(txnId, this, isolationLevel);
|
||||
_activeTransactions[txnId] = transaction;
|
||||
return transaction;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins a new transaction asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The started transaction.</returns>
|
||||
public async Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, CancellationToken ct = default)
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var txnId = _nextTransactionId++;
|
||||
var transaction = new Transaction(txnId, this, isolationLevel);
|
||||
_activeTransactions[txnId] = transaction;
|
||||
return transaction;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the specified transaction.
|
||||
/// </summary>
|
||||
/// <param name="transaction">The transaction to commit.</param>
|
||||
public void CommitTransaction(Transaction transaction)
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
||||
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
||||
|
||||
// 1. Prepare (Write to WAL)
|
||||
// In a fuller 2PC, this would be separate. Here we do it as part of commit.
|
||||
if (!PrepareTransaction(transaction.TransactionId))
|
||||
throw new IOException("Failed to write transaction to WAL");
|
||||
|
||||
// 2. Commit (Write commit record, flush, move to cache)
|
||||
// Use core commit path to avoid re-entering _commitLock.
|
||||
CommitTransactionCore(transaction.TransactionId);
|
||||
|
||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the specified transaction asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="transaction">The transaction to commit.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
public async Task CommitTransactionAsync(Transaction transaction, CancellationToken ct = default)
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
||||
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
||||
|
||||
// 1. Prepare (Write to WAL)
|
||||
if (!await PrepareTransactionAsync(transaction.TransactionId, ct))
|
||||
throw new IOException("Failed to write transaction to WAL");
|
||||
|
||||
// 2. Commit (Write commit record, flush, move to cache)
|
||||
// Use core commit path to avoid re-entering _commitLock.
|
||||
await CommitTransactionCoreAsync(transaction.TransactionId, ct);
|
||||
|
||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the specified transaction.
|
||||
/// </summary>
|
||||
/// <param name="transaction">The transaction to roll back.</param>
|
||||
public void RollbackTransaction(Transaction transaction)
|
||||
{
|
||||
RollbackTransaction(transaction.TransactionId);
|
||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||
}
|
||||
|
||||
// Rollback doesn't usually require async logic unless logging abort record is async,
|
||||
// but for consistency we might consider it. For now, sync is fine as it's not the happy path bottleneck.
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Prepares a transaction: writes all changes to WAL but doesn't commit yet.
|
||||
/// Part of 2-Phase Commit protocol.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">Transaction ID</param>
|
||||
/// <param name="writeSet">All writes to record in WAL</param>
|
||||
/// <returns>True if preparation succeeded</returns>
|
||||
public bool PrepareTransaction(ulong transactionId)
|
||||
{
|
||||
try
|
||||
{
|
||||
_wal.WriteBeginRecord(transactionId);
|
||||
|
||||
foreach (var walEntry in _walCache[transactionId])
|
||||
{
|
||||
_wal.WriteDataRecord(transactionId, walEntry.Key, walEntry.Value);
|
||||
}
|
||||
|
||||
_wal.Flush(); // Ensure WAL is persisted
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// TODO: Log error?
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prepares a transaction asynchronously by writing pending changes to the WAL.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns><see langword="true"/> if preparation succeeds; otherwise, <see langword="false"/>.</returns>
|
||||
public async Task<bool> PrepareTransactionAsync(ulong transactionId, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
||||
|
||||
if (_walCache.TryGetValue(transactionId, out var changes))
|
||||
{
|
||||
foreach (var walEntry in changes)
|
||||
{
|
||||
await _wal.WriteDataRecordAsync(transactionId, walEntry.Key, walEntry.Value, ct);
|
||||
}
|
||||
}
|
||||
|
||||
await _wal.FlushAsync(ct); // Ensure WAL is persisted
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits a transaction:
|
||||
/// 1. Writes all changes to WAL (for durability)
|
||||
/// 2. Writes commit record
|
||||
/// 3. Flushes WAL to disk
|
||||
/// 4. Moves pages from cache to WAL index (for future reads)
|
||||
/// 5. Clears WAL cache
|
||||
/// </summary>
|
||||
/// <param name="transactionId">Transaction to commit</param>
|
||||
/// <param name="writeSet">All writes performed in this transaction (unused, kept for compatibility)</param>
|
||||
public void CommitTransaction(ulong transactionId)
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
CommitTransactionCore(transactionId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void CommitTransactionCore(ulong transactionId)
|
||||
{
|
||||
// Get ALL pages from WAL cache (includes both data and index pages)
|
||||
if (!_walCache.TryGetValue(transactionId, out var pages))
|
||||
{
|
||||
// No writes for this transaction, just write commit record
|
||||
_wal.WriteCommitRecord(transactionId);
|
||||
_wal.Flush();
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Write all changes to WAL (from cache, not writeSet!)
|
||||
_wal.WriteBeginRecord(transactionId);
|
||||
|
||||
foreach (var (pageId, data) in pages)
|
||||
{
|
||||
_wal.WriteDataRecord(transactionId, pageId, data);
|
||||
}
|
||||
|
||||
// 2. Write commit record and flush
|
||||
_wal.WriteCommitRecord(transactionId);
|
||||
_wal.Flush(); // Durability: ensure WAL is on disk
|
||||
|
||||
// 3. Move pages from cache to WAL index (for reads)
|
||||
_walCache.TryRemove(transactionId, out _);
|
||||
foreach (var kvp in pages)
|
||||
{
|
||||
_walIndex[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
// Auto-checkpoint if WAL grows too large
|
||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
||||
{
|
||||
CheckpointInternal();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits a prepared transaction asynchronously by identifier.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
public async Task CommitTransactionAsync(ulong transactionId, CancellationToken ct = default)
|
||||
{
|
||||
await _commitLock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
await CommitTransactionCoreAsync(transactionId, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CommitTransactionCoreAsync(ulong transactionId, CancellationToken ct)
|
||||
{
|
||||
// Get ALL pages from WAL cache (includes both data and index pages)
|
||||
if (!_walCache.TryGetValue(transactionId, out var pages))
|
||||
{
|
||||
// No writes for this transaction, just write commit record
|
||||
await _wal.WriteCommitRecordAsync(transactionId, ct);
|
||||
await _wal.FlushAsync(ct);
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. Write all changes to WAL (from cache, not writeSet!)
|
||||
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
||||
|
||||
foreach (var (pageId, data) in pages)
|
||||
{
|
||||
await _wal.WriteDataRecordAsync(transactionId, pageId, data, ct);
|
||||
}
|
||||
|
||||
// 2. Write commit record and flush
|
||||
await _wal.WriteCommitRecordAsync(transactionId, ct);
|
||||
await _wal.FlushAsync(ct); // Durability: ensure WAL is on disk
|
||||
|
||||
// 3. Move pages from cache to WAL index (for reads)
|
||||
_walCache.TryRemove(transactionId, out _);
|
||||
foreach (var kvp in pages)
|
||||
{
|
||||
_walIndex[kvp.Key] = kvp.Value;
|
||||
}
|
||||
|
||||
// Auto-checkpoint if WAL grows too large
|
||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
||||
{
|
||||
// Checkpoint might be sync or async. For now sync inside the lock is "safe" but blocking.
|
||||
// Ideally this should be async too.
|
||||
CheckpointInternal();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks a transaction as committed after WAL writes.
|
||||
/// Used for 2PC: after Prepare() writes to WAL, this finalizes the commit.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">Transaction to mark committed</param>
|
||||
public void MarkTransactionCommitted(ulong transactionId)
|
||||
{
|
||||
_commitLock.Wait();
|
||||
try
|
||||
{
|
||||
_wal.WriteCommitRecord(transactionId);
|
||||
_wal.Flush();
|
||||
|
||||
// Move from cache to WAL index
|
||||
if (_walCache.TryRemove(transactionId, out var pages))
|
||||
{
|
||||
foreach (var kvp in pages)
|
||||
{
|
||||
_walIndex[kvp.Key] = kvp.Value;
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-checkpoint if WAL grows too large
|
||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
||||
{
|
||||
CheckpointInternal();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_commitLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back a transaction: discards all uncommitted changes.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">Transaction to rollback</param>
|
||||
public void RollbackTransaction(ulong transactionId)
|
||||
{
|
||||
_walCache.TryRemove(transactionId, out _);
|
||||
_wal.WriteAbortRecord(transactionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an abort record for the specified transaction.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
internal void WriteAbortRecord(ulong transactionId)
|
||||
{
|
||||
_wal.WriteAbortRecord(transactionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of active transactions (diagnostics).
|
||||
/// </summary>
|
||||
public int ActiveTransactionCount => _walCache.Count;
|
||||
}
|
||||
184
src/CBDD.Core/Storage/StorageEngine.cs
Executable file
184
src/CBDD.Core/Storage/StorageEngine.cs
Executable file
@@ -0,0 +1,184 @@
|
||||
using System.Collections.Concurrent;
|
||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Central storage engine managing page-based storage with WAL for durability.
|
||||
///
|
||||
/// Architecture (WAL-based like SQLite/PostgreSQL):
|
||||
/// - PageFile: Committed baseline (persistent on disk)
|
||||
/// - WAL Cache: Uncommitted transaction writes (in-memory)
|
||||
/// - Read: PageFile + WAL cache overlay (for Read Your Own Writes)
|
||||
/// - Commit: Flush to WAL, clear cache
|
||||
/// - Checkpoint: Merge WAL ? PageFile periodically
|
||||
/// </summary>
|
||||
public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
||||
{
|
||||
private readonly PageFile _pageFile;
|
||||
private readonly WriteAheadLog _wal;
|
||||
private readonly CompressionOptions _compressionOptions;
|
||||
private readonly CompressionService _compressionService;
|
||||
private readonly CompressionTelemetry _compressionTelemetry;
|
||||
private readonly StorageFormatMetadata _storageFormatMetadata;
|
||||
private readonly MaintenanceOptions _maintenanceOptions;
|
||||
private CDC.ChangeStreamDispatcher? _cdc;
|
||||
|
||||
// WAL cache: TransactionId → (PageId → PageData)
|
||||
// Stores uncommitted writes for "Read Your Own Writes" isolation
|
||||
private readonly ConcurrentDictionary<ulong, ConcurrentDictionary<uint, byte[]>> _walCache;
|
||||
|
||||
// WAL index cache: PageId → PageData (from latest committed transaction)
|
||||
// Lazily populated on first read after commit
|
||||
private readonly ConcurrentDictionary<uint, byte[]> _walIndex;
|
||||
|
||||
// Global lock for commit/checkpoint synchronization
|
||||
private readonly SemaphoreSlim _commitLock = new(1, 1);
|
||||
|
||||
// Transaction Management
|
||||
private readonly ConcurrentDictionary<ulong, Transaction> _activeTransactions;
|
||||
private ulong _nextTransactionId;
|
||||
|
||||
private const long MaxWalSize = 4 * 1024 * 1024; // 4MB
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="StorageEngine"/> class.
|
||||
/// </summary>
|
||||
/// <param name="databasePath">The database file path.</param>
|
||||
/// <param name="config">The page file configuration.</param>
|
||||
public StorageEngine(
|
||||
string databasePath,
|
||||
PageFileConfig config,
|
||||
CompressionOptions? compressionOptions = null,
|
||||
MaintenanceOptions? maintenanceOptions = null)
|
||||
{
|
||||
_compressionOptions = CompressionOptions.Normalize(compressionOptions);
|
||||
_compressionService = new CompressionService();
|
||||
_compressionTelemetry = new CompressionTelemetry();
|
||||
_maintenanceOptions = maintenanceOptions ?? new MaintenanceOptions();
|
||||
|
||||
// Auto-derive WAL path
|
||||
var walPath = Path.ChangeExtension(databasePath, ".wal");
|
||||
|
||||
// Initialize storage infrastructure
|
||||
_pageFile = new PageFile(databasePath, config);
|
||||
_pageFile.Open();
|
||||
|
||||
_wal = new WriteAheadLog(walPath);
|
||||
_walCache = new ConcurrentDictionary<ulong, ConcurrentDictionary<uint, byte[]>>();
|
||||
_walIndex = new ConcurrentDictionary<uint, byte[]>();
|
||||
_activeTransactions = new ConcurrentDictionary<ulong, Transaction>();
|
||||
_nextTransactionId = 1;
|
||||
_storageFormatMetadata = InitializeStorageFormatMetadata();
|
||||
|
||||
// Recover from WAL if exists (crash recovery or resume after close)
|
||||
// This replays any committed transactions not yet checkpointed
|
||||
if (_wal.GetCurrentSize() > 0)
|
||||
{
|
||||
Recover();
|
||||
}
|
||||
|
||||
_ = ResumeCompactionIfNeeded();
|
||||
|
||||
InitializeDictionary();
|
||||
TryRunStartupMaintenance();
|
||||
|
||||
// Create and start checkpoint manager
|
||||
// _checkpointManager = new Transactions.CheckpointManager(this);
|
||||
// _checkpointManager.StartAutoCheckpoint();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Page size for this storage engine
|
||||
/// </summary>
|
||||
public int PageSize => _pageFile.PageSize;
|
||||
|
||||
/// <summary>
|
||||
/// Compression options for this engine instance.
|
||||
/// </summary>
|
||||
public CompressionOptions CompressionOptions => _compressionOptions;
|
||||
|
||||
/// <summary>
|
||||
/// Compression codec service for payload roundtrip operations.
|
||||
/// </summary>
|
||||
public CompressionService CompressionService => _compressionService;
|
||||
|
||||
/// <summary>
|
||||
/// Compression telemetry counters for this engine instance.
|
||||
/// </summary>
|
||||
public CompressionTelemetry CompressionTelemetry => _compressionTelemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||
/// </summary>
|
||||
public CompressionStats GetCompressionStats() => _compressionTelemetry.GetSnapshot();
|
||||
|
||||
internal StorageFormatMetadata StorageFormatMetadata => _storageFormatMetadata;
|
||||
|
||||
/// <summary>
|
||||
/// Checks if a page is currently being modified by another active transaction.
|
||||
/// This is used to implement pessimistic locking for page allocation/selection.
|
||||
/// </summary>
|
||||
/// <param name="pageId">The page identifier to check.</param>
|
||||
/// <param name="excludingTxId">The transaction identifier to exclude from the check.</param>
|
||||
/// <returns><see langword="true"/> if another transaction holds the page; otherwise, <see langword="false"/>.</returns>
|
||||
public bool IsPageLocked(uint pageId, ulong excludingTxId)
|
||||
{
|
||||
foreach (var kvp in _walCache)
|
||||
{
|
||||
var txId = kvp.Key;
|
||||
if (txId == excludingTxId) continue;
|
||||
|
||||
var txnPages = kvp.Value;
|
||||
if (txnPages.ContainsKey(pageId))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the storage engine and closes WAL.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
// 1. Rollback any active transactions
|
||||
if (_activeTransactions != null)
|
||||
{
|
||||
foreach (var txn in _activeTransactions.Values)
|
||||
{
|
||||
try
|
||||
{
|
||||
RollbackTransaction(txn.TransactionId);
|
||||
}
|
||||
catch { /* Ignore errors during dispose */ }
|
||||
}
|
||||
_activeTransactions.Clear();
|
||||
}
|
||||
|
||||
// 2. Close WAL and PageFile
|
||||
_wal?.Dispose();
|
||||
_pageFile?.Dispose();
|
||||
_maintenanceGate?.Dispose();
|
||||
_commitLock?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers the change stream dispatcher used for CDC notifications.
|
||||
/// </summary>
|
||||
/// <param name="cdc">The change stream dispatcher instance.</param>
|
||||
internal void RegisterCdc(CDC.ChangeStreamDispatcher cdc)
|
||||
{
|
||||
_cdc = cdc;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the registered change stream dispatcher, if available.
|
||||
/// </summary>
|
||||
internal CDC.ChangeStreamDispatcher? Cdc => _cdc;
|
||||
void IStorageEngine.RegisterCdc(CDC.ChangeStreamDispatcher cdc) => RegisterCdc(cdc);
|
||||
CDC.ChangeStreamDispatcher? IStorageEngine.Cdc => _cdc;
|
||||
CompressionOptions IStorageEngine.CompressionOptions => _compressionOptions;
|
||||
CompressionService IStorageEngine.CompressionService => _compressionService;
|
||||
CompressionTelemetry IStorageEngine.CompressionTelemetry => _compressionTelemetry;
|
||||
}
|
||||
174
src/CBDD.Core/Storage/VectorPage.cs
Executable file
174
src/CBDD.Core/Storage/VectorPage.cs
Executable file
@@ -0,0 +1,174 @@
|
||||
using System.Runtime.InteropServices;
|
||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Page for storing HNSW Vector Index nodes.
|
||||
/// Each page stores a fixed number of nodes based on vector dimensions and M.
|
||||
/// </summary>
|
||||
public struct VectorPage
|
||||
{
|
||||
// Layout:
|
||||
// [PageHeader (32)]
|
||||
// [Dimensions (4)]
|
||||
// [MaxM (4)]
|
||||
// [NodeSize (4)]
|
||||
// [NodeCount (4)]
|
||||
// [Nodes Data (Contiguous)...]
|
||||
|
||||
private const int DimensionsOffset = 32;
|
||||
private const int MaxMOffset = 36;
|
||||
private const int NodeSizeOffset = 40;
|
||||
private const int NodeCountOffset = 44;
|
||||
private const int DataOffset = 48;
|
||||
|
||||
/// <summary>
|
||||
/// Increments the node count stored in the vector page header.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
public static void IncrementNodeCount(Span<byte> page)
|
||||
{
|
||||
int count = GetNodeCount(page);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), count + 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a vector page with header metadata and sizing information.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="pageId">The page identifier.</param>
|
||||
/// <param name="dimensions">The vector dimensionality.</param>
|
||||
/// <param name="maxM">The HNSW max connections parameter.</param>
|
||||
public static void Initialize(Span<byte> page, uint pageId, int dimensions, int maxM)
|
||||
{
|
||||
var header = new PageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Vector,
|
||||
FreeBytes = (ushort)(page.Length - DataOffset),
|
||||
NextPageId = 0,
|
||||
TransactionId = 0
|
||||
};
|
||||
header.WriteTo(page);
|
||||
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(DimensionsOffset), dimensions);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(MaxMOffset), maxM);
|
||||
|
||||
// Node Size Calculation:
|
||||
// Location (6) + MaxLevel (1) + Vector (dim * 4) + Links (maxM * 10 * 6) -- estimating 10 levels for simplicity
|
||||
// Better: Node size is variable? No, let's keep it fixed per index configuration to avoid fragmentation.
|
||||
// HNSW standard: level 0 has 2*M links, levels > 0 have M links.
|
||||
// Max level is typically < 16. Let's reserve space for 16 levels.
|
||||
int nodeSize = 6 + 1 + (dimensions * 4) + (maxM * (2 + 15) * 6);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeSizeOffset), nodeSize);
|
||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of nodes currently stored in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The node count.</returns>
|
||||
public static int GetNodeCount(ReadOnlySpan<byte> page) =>
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeCountOffset));
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured node size for the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The node size in bytes.</returns>
|
||||
public static int GetNodeSize(ReadOnlySpan<byte> page) =>
|
||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeSizeOffset));
|
||||
|
||||
/// <summary>
|
||||
/// Gets the maximum number of nodes that can fit in the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <returns>The maximum node count.</returns>
|
||||
public static int GetMaxNodes(ReadOnlySpan<byte> page) =>
|
||||
(page.Length - DataOffset) / GetNodeSize(page);
|
||||
|
||||
/// <summary>
|
||||
/// Writes a node to the page at the specified index.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||
/// <param name="loc">The document location for the node.</param>
|
||||
/// <param name="maxLevel">The maximum graph level for the node.</param>
|
||||
/// <param name="vector">The vector values to store.</param>
|
||||
/// <param name="dimensions">The vector dimensionality.</param>
|
||||
public static void WriteNode(Span<byte> page, int nodeIndex, DocumentLocation loc, int maxLevel, ReadOnlySpan<float> vector, int dimensions)
|
||||
{
|
||||
int nodeSize = GetNodeSize(page);
|
||||
int offset = DataOffset + (nodeIndex * nodeSize);
|
||||
var nodeSpan = page.Slice(offset, nodeSize);
|
||||
|
||||
// 1. Document Location
|
||||
loc.WriteTo(nodeSpan.Slice(0, 6));
|
||||
|
||||
// 2. Max Level
|
||||
nodeSpan[6] = (byte)maxLevel;
|
||||
|
||||
// 3. Vector
|
||||
var vectorSpan = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, dimensions * 4));
|
||||
vector.CopyTo(vectorSpan);
|
||||
|
||||
// 4. Links (initialize with 0/empty)
|
||||
// Links follow the vector. Level 0: 2*M links, Level 1..15: M links.
|
||||
// For now, just ensure it's cleared or handled by the indexer.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads node metadata and vector data from the page.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||
/// <param name="loc">When this method returns, contains the node document location.</param>
|
||||
/// <param name="maxLevel">When this method returns, contains the node max level.</param>
|
||||
/// <param name="vector">The destination span for vector values.</param>
|
||||
public static void ReadNodeData(ReadOnlySpan<byte> page, int nodeIndex, out DocumentLocation loc, out int maxLevel, Span<float> vector)
|
||||
{
|
||||
int nodeSize = GetNodeSize(page);
|
||||
int offset = DataOffset + (nodeIndex * nodeSize);
|
||||
var nodeSpan = page.Slice(offset, nodeSize);
|
||||
|
||||
loc = DocumentLocation.ReadFrom(nodeSpan.Slice(0, 6));
|
||||
maxLevel = nodeSpan[6];
|
||||
|
||||
var vectorSource = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, vector.Length * 4));
|
||||
vectorSource.CopyTo(vector);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the span that stores links for a node at a specific level.
|
||||
/// </summary>
|
||||
/// <param name="page">The page buffer.</param>
|
||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||
/// <param name="level">The graph level.</param>
|
||||
/// <param name="dimensions">The vector dimensionality.</param>
|
||||
/// <param name="maxM">The HNSW max connections parameter.</param>
|
||||
/// <returns>The span representing the link storage region.</returns>
|
||||
public static Span<byte> GetLinksSpan(Span<byte> page, int nodeIndex, int level, int dimensions, int maxM)
|
||||
{
|
||||
int nodeSize = GetNodeSize(page);
|
||||
int nodeOffset = DataOffset + (nodeIndex * nodeSize);
|
||||
|
||||
// Link offset: Location(6) + MaxLevel(1) + Vector(dim*4)
|
||||
int linkBaseOffset = nodeOffset + 7 + (dimensions * 4);
|
||||
|
||||
int levelOffset;
|
||||
if (level == 0)
|
||||
{
|
||||
levelOffset = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Level 0 has 2*M links
|
||||
levelOffset = (2 * maxM * 6) + ((level - 1) * maxM * 6);
|
||||
}
|
||||
|
||||
int count = (level == 0) ? (2 * maxM) : maxM;
|
||||
return page.Slice(linkBaseOffset + levelOffset, count * 6);
|
||||
}
|
||||
}
|
||||
32
src/CBDD.Core/Transactions/CheckpointMode.cs
Executable file
32
src/CBDD.Core/Transactions/CheckpointMode.cs
Executable file
@@ -0,0 +1,32 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// Defines checkpoint modes for WAL (Write-Ahead Log) checkpointing.
|
||||
/// Similar to SQLite's checkpoint strategies.
|
||||
/// </summary>
|
||||
public enum CheckpointMode
|
||||
{
|
||||
/// <summary>
|
||||
/// Passive checkpoint: Non-blocking, best-effort transfer from WAL to database.
|
||||
/// Does not wait for readers or writers. May not checkpoint all frames.
|
||||
/// </summary>
|
||||
Passive = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Full checkpoint: Waits for concurrent readers/writers, then checkpoints all
|
||||
/// committed transactions from WAL to database. Blocks until complete.
|
||||
/// </summary>
|
||||
Full = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Truncate checkpoint: Same as Full, but also truncates the WAL file after
|
||||
/// successful checkpoint. Use this to reclaim disk space.
|
||||
/// </summary>
|
||||
Truncate = 2,
|
||||
|
||||
/// <summary>
|
||||
/// Restart checkpoint: Truncates WAL and restarts with a new WAL file.
|
||||
/// Forces a fresh start. Most aggressive mode.
|
||||
/// </summary>
|
||||
Restart = 3
|
||||
}
|
||||
66
src/CBDD.Core/Transactions/ITransaction.cs
Executable file
66
src/CBDD.Core/Transactions/ITransaction.cs
Executable file
@@ -0,0 +1,66 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// Public interface for database transactions.
|
||||
/// Allows user-controlled transaction boundaries for batch operations.
|
||||
/// </summary>
|
||||
/// <example>
|
||||
/// using (var txn = collection.BeginTransaction())
|
||||
/// {
|
||||
/// collection.Insert(entity1, txn);
|
||||
/// collection.Insert(entity2, txn);
|
||||
/// txn.Commit();
|
||||
/// }
|
||||
/// </example>
|
||||
public interface ITransaction : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique transaction identifier
|
||||
/// </summary>
|
||||
ulong TransactionId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Current state of the transaction
|
||||
/// </summary>
|
||||
TransactionState State { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Commits the transaction, making all changes permanent.
|
||||
/// Must be called before Dispose() to persist changes.
|
||||
/// </summary>
|
||||
void Commit();
|
||||
|
||||
/// <summary>
|
||||
/// Asynchronously commits the transaction, making all changes permanent.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
Task CommitAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the transaction, discarding all changes.
|
||||
/// Called automatically on Dispose() if Commit() was not called.
|
||||
/// </summary>
|
||||
void Rollback();
|
||||
|
||||
/// <summary>
|
||||
/// Adds a write operation to the current batch or transaction.
|
||||
/// </summary>
|
||||
/// <param name="operation">The write operation to add. Cannot be null.</param>
|
||||
void AddWrite(WriteOperation operation);
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the object for use by performing any necessary initialization or setup.
|
||||
/// </summary>
|
||||
/// <returns>true if the preparation was successful; otherwise, false.</returns>
|
||||
bool Prepare();
|
||||
|
||||
/// <summary>
|
||||
/// Event triggered when the transaction acts rollback.
|
||||
/// Useful for restoring in-memory state (like ID maps).
|
||||
/// </summary>
|
||||
event Action? OnRollback;
|
||||
}
|
||||
26
src/CBDD.Core/Transactions/ITransactionHolder.cs
Executable file
26
src/CBDD.Core/Transactions/ITransactionHolder.cs
Executable file
@@ -0,0 +1,26 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// Defines a contract for managing and providing access to the current transaction context.
|
||||
/// </summary>
|
||||
/// <remarks>Implementations of this interface are responsible for tracking the current transaction and starting a
|
||||
/// new one if none exists. This is typically used in scenarios where transactional consistency is required across
|
||||
/// multiple operations.</remarks>
|
||||
public interface ITransactionHolder
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the current transaction if one exists; otherwise, starts a new transaction.
|
||||
/// </summary>
|
||||
/// <remarks>Use this method to ensure that a transaction context is available for the current operation.
|
||||
/// If a transaction is already in progress, it is returned; otherwise, a new transaction is started and returned.
|
||||
/// The caller is responsible for managing the transaction's lifetime as appropriate.</remarks>
|
||||
/// <returns>An <see cref="ITransaction"/> representing the current transaction, or a new transaction if none is active.</returns>
|
||||
ITransaction GetCurrentTransactionOrStart();
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current transaction if one exists; otherwise, starts a new transaction asynchronously.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation. The task result contains an <see cref="ITransaction"/>
|
||||
/// representing the current or newly started transaction.</returns>
|
||||
Task<ITransaction> GetCurrentTransactionOrStartAsync();
|
||||
}
|
||||
270
src/CBDD.Core/Transactions/Transaction.cs
Executable file
270
src/CBDD.Core/Transactions/Transaction.cs
Executable file
@@ -0,0 +1,270 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a transaction with ACID properties.
|
||||
/// Uses MVCC (Multi-Version Concurrency Control) for isolation.
|
||||
/// </summary>
|
||||
public sealed class Transaction : ITransaction
|
||||
{
|
||||
private readonly ulong _transactionId;
|
||||
private readonly IsolationLevel _isolationLevel;
|
||||
private readonly DateTime _startTime;
|
||||
private readonly StorageEngine _storage;
|
||||
private readonly List<CDC.InternalChangeEvent> _pendingChanges = new();
|
||||
private TransactionState _state;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new transaction.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The unique transaction identifier.</param>
|
||||
/// <param name="storage">The storage engine used by this transaction.</param>
|
||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||
public Transaction(ulong transactionId,
|
||||
StorageEngine storage,
|
||||
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
||||
{
|
||||
_transactionId = transactionId;
|
||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||
_isolationLevel = isolationLevel;
|
||||
_startTime = DateTime.UtcNow;
|
||||
_state = TransactionState.Active;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a pending CDC change to be published after commit.
|
||||
/// </summary>
|
||||
/// <param name="change">The change event to buffer.</param>
|
||||
internal void AddChange(CDC.InternalChangeEvent change)
|
||||
{
|
||||
_pendingChanges.Add(change);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the unique transaction identifier.
|
||||
/// </summary>
|
||||
public ulong TransactionId => _transactionId;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current transaction state.
|
||||
/// </summary>
|
||||
public TransactionState State => _state;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configured transaction isolation level.
|
||||
/// </summary>
|
||||
public IsolationLevel IsolationLevel => _isolationLevel;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the UTC start time of the transaction.
|
||||
/// </summary>
|
||||
public DateTime StartTime => _startTime;
|
||||
|
||||
/// <summary>
|
||||
/// Adds a write operation to the transaction's write set.
|
||||
/// NOTE: Makes a defensive copy of the data to ensure memory safety.
|
||||
/// This allocation is necessary because the caller may return the buffer to a pool.
|
||||
/// </summary>
|
||||
/// <param name="operation">The write operation to add.</param>
|
||||
public void AddWrite(WriteOperation operation)
|
||||
{
|
||||
if (_state != TransactionState.Active)
|
||||
throw new InvalidOperationException($"Cannot add writes to transaction in state {_state}");
|
||||
|
||||
// Defensive copy: necessary to prevent use-after-return if caller uses pooled buffers
|
||||
byte[] ownedCopy = operation.NewValue.ToArray();
|
||||
// StorageEngine gestisce tutte le scritture transazionali
|
||||
_storage.WritePage(operation.PageId, _transactionId, ownedCopy);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the transaction for commit (2PC first phase)
|
||||
/// </summary>
|
||||
public bool Prepare()
|
||||
{
|
||||
if (_state != TransactionState.Active)
|
||||
return false;
|
||||
|
||||
_state = TransactionState.Preparing;
|
||||
|
||||
// StorageEngine handles WAL writes
|
||||
return _storage.PrepareTransaction(_transactionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits the transaction.
|
||||
/// Writes to WAL for durability and moves data to committed buffer.
|
||||
/// Pages remain in memory until CheckpointManager writes them to disk.
|
||||
/// </summary>
|
||||
public void Commit()
|
||||
{
|
||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
||||
|
||||
// StorageEngine handles WAL writes and buffer management
|
||||
_storage.CommitTransaction(_transactionId);
|
||||
|
||||
_state = TransactionState.Committed;
|
||||
|
||||
// Publish CDC events after successful commit
|
||||
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
||||
{
|
||||
foreach (var change in _pendingChanges)
|
||||
{
|
||||
_storage.Cdc.Publish(change);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asynchronously commits the transaction.
|
||||
/// </summary>
|
||||
/// <param name="ct">A cancellation token.</param>
|
||||
public async Task CommitAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
||||
|
||||
// StorageEngine handles WAL writes and buffer management
|
||||
await _storage.CommitTransactionAsync(_transactionId, ct);
|
||||
|
||||
_state = TransactionState.Committed;
|
||||
|
||||
// Publish CDC events after successful commit
|
||||
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
||||
{
|
||||
foreach (var change in _pendingChanges)
|
||||
{
|
||||
_storage.Cdc.Publish(change);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marks the transaction as committed without writing to PageFile.
|
||||
/// Used by TransactionManager with lazy checkpointing.
|
||||
/// </summary>
|
||||
internal void MarkCommitted()
|
||||
{
|
||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
||||
|
||||
// StorageEngine marks transaction as committed and moves to committed buffer
|
||||
_storage.MarkTransactionCommitted(_transactionId);
|
||||
|
||||
_state = TransactionState.Committed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the transaction (discards all writes)
|
||||
/// </summary>
|
||||
public event Action? OnRollback;
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the transaction and discards pending writes.
|
||||
/// </summary>
|
||||
public void Rollback()
|
||||
{
|
||||
if (_state == TransactionState.Committed)
|
||||
throw new InvalidOperationException("Cannot rollback committed transaction");
|
||||
|
||||
_pendingChanges.Clear();
|
||||
_storage.RollbackTransaction(_transactionId);
|
||||
_state = TransactionState.Aborted;
|
||||
|
||||
OnRollback?.Invoke();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases transaction resources and rolls back if still active.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
|
||||
if (_state == TransactionState.Active || _state == TransactionState.Preparing)
|
||||
{
|
||||
// Auto-rollback if not committed
|
||||
Rollback();
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a write operation in a transaction.
|
||||
/// Optimized to avoid allocations by using ReadOnlyMemory instead of byte[].
|
||||
/// </summary>
|
||||
public struct WriteOperation
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the identifier of the affected document.
|
||||
/// </summary>
|
||||
public ObjectId DocumentId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the new serialized value.
|
||||
/// </summary>
|
||||
public ReadOnlyMemory<byte> NewValue { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the target page identifier.
|
||||
/// </summary>
|
||||
public uint PageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the operation type.
|
||||
/// </summary>
|
||||
public OperationType Type { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new write operation.
|
||||
/// </summary>
|
||||
/// <param name="documentId">The identifier of the affected document.</param>
|
||||
/// <param name="newValue">The new serialized value.</param>
|
||||
/// <param name="pageId">The target page identifier.</param>
|
||||
/// <param name="type">The operation type.</param>
|
||||
public WriteOperation(ObjectId documentId, ReadOnlyMemory<byte> newValue, uint pageId, OperationType type)
|
||||
{
|
||||
DocumentId = documentId;
|
||||
NewValue = newValue;
|
||||
PageId = pageId;
|
||||
Type = type;
|
||||
}
|
||||
|
||||
// Backward compatibility constructor
|
||||
/// <summary>
|
||||
/// Initializes a new write operation from a byte array payload.
|
||||
/// </summary>
|
||||
/// <param name="documentId">The identifier of the affected document.</param>
|
||||
/// <param name="newValue">The new serialized value.</param>
|
||||
/// <param name="pageId">The target page identifier.</param>
|
||||
/// <param name="type">The operation type.</param>
|
||||
public WriteOperation(ObjectId documentId, byte[] newValue, uint pageId, OperationType type)
|
||||
{
|
||||
DocumentId = documentId;
|
||||
NewValue = newValue;
|
||||
PageId = pageId;
|
||||
Type = type;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Type of write operation
|
||||
/// </summary>
|
||||
public enum OperationType : byte
|
||||
{
|
||||
Insert = 1,
|
||||
Update = 2,
|
||||
Delete = 3,
|
||||
AllocatePage = 4
|
||||
}
|
||||
37
src/CBDD.Core/Transactions/TransactionState.cs
Executable file
37
src/CBDD.Core/Transactions/TransactionState.cs
Executable file
@@ -0,0 +1,37 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// Transaction states
|
||||
/// </summary>
|
||||
public enum TransactionState : byte
|
||||
{
|
||||
/// <summary>Transaction is active and can accept operations</summary>
|
||||
Active = 1,
|
||||
|
||||
/// <summary>Transaction is preparing to commit</summary>
|
||||
Preparing = 2,
|
||||
|
||||
/// <summary>Transaction committed successfully</summary>
|
||||
Committed = 3,
|
||||
|
||||
/// <summary>Transaction was rolled back</summary>
|
||||
Aborted = 4
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Transaction isolation levels
|
||||
/// </summary>
|
||||
public enum IsolationLevel : byte
|
||||
{
|
||||
/// <summary>Read uncommitted data</summary>
|
||||
ReadUncommitted = 0,
|
||||
|
||||
/// <summary>Read only committed data (default)</summary>
|
||||
ReadCommitted = 1,
|
||||
|
||||
/// <summary>Repeatable reads</summary>
|
||||
RepeatableRead = 2,
|
||||
|
||||
/// <summary>Serializable (full isolation)</summary>
|
||||
Serializable = 3
|
||||
}
|
||||
572
src/CBDD.Core/Transactions/WriteAheadLog.cs
Executable file
572
src/CBDD.Core/Transactions/WriteAheadLog.cs
Executable file
@@ -0,0 +1,572 @@
|
||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||
|
||||
/// <summary>
|
||||
/// WAL record types
|
||||
/// </summary>
|
||||
public enum WalRecordType : byte
|
||||
{
|
||||
Begin = 1,
|
||||
Write = 2,
|
||||
Commit = 3,
|
||||
Abort = 4,
|
||||
Checkpoint = 5
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write-Ahead Log (WAL) for durability and recovery.
|
||||
/// All changes are logged before being applied.
|
||||
/// </summary>
|
||||
public sealed class WriteAheadLog : IDisposable
|
||||
{
|
||||
private readonly string _walPath;
|
||||
private FileStream? _walStream;
|
||||
private readonly SemaphoreSlim _lock = new(1, 1);
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="WriteAheadLog"/> class.
|
||||
/// </summary>
|
||||
/// <param name="walPath">The file path of the write-ahead log.</param>
|
||||
public WriteAheadLog(string walPath)
|
||||
{
|
||||
_walPath = walPath ?? throw new ArgumentNullException(nameof(walPath));
|
||||
|
||||
_walStream = new FileStream(
|
||||
_walPath,
|
||||
FileMode.OpenOrCreate,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None, // Exclusive access like PageFile
|
||||
bufferSize: 64 * 1024); // 64KB buffer for better sequential write performance
|
||||
// REMOVED FileOptions.WriteThrough for SQLite-style lazy checkpointing
|
||||
// Durability is ensured by explicit Flush() calls
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a begin transaction record
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
public void WriteBeginRecord(ulong transactionId)
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
WriteBeginRecordInternal(transactionId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a begin transaction record asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||
public async ValueTask WriteBeginRecordAsync(ulong transactionId, CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
// Use ArrayPool for async I/O compatibility (cannot use stackalloc with async)
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(17);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Begin;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), transactionId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, 17), ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteBeginRecordInternal(ulong transactionId)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[17]; // type(1) + txnId(8) + timestamp(8)
|
||||
buffer[0] = (byte)WalRecordType.Begin;
|
||||
BitConverter.TryWriteBytes(buffer[1..9], transactionId);
|
||||
BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
_walStream!.Write(buffer);
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes a commit record
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Writes a commit record
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
public void WriteCommitRecord(ulong transactionId)
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
WriteCommitRecordInternal(transactionId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a commit record asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||
public async ValueTask WriteCommitRecordAsync(ulong transactionId, CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(17);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Commit;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), transactionId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, 17), ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteCommitRecordInternal(ulong transactionId)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[17]; // type(1) + txnId(8) + timestamp(8)
|
||||
buffer[0] = (byte)WalRecordType.Commit;
|
||||
BitConverter.TryWriteBytes(buffer[1..9], transactionId);
|
||||
BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
_walStream!.Write(buffer);
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes an abort record
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Writes an abort record
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
public void WriteAbortRecord(ulong transactionId)
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
WriteAbortRecordInternal(transactionId);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes an abort record asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||
public async ValueTask WriteAbortRecordAsync(ulong transactionId, CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(17);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Abort;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), transactionId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 8), DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, 17), ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteAbortRecordInternal(ulong transactionId)
|
||||
{
|
||||
Span<byte> buffer = stackalloc byte[17]; // type(1) + txnId(8) + timestamp(8)
|
||||
buffer[0] = (byte)WalRecordType.Abort;
|
||||
BitConverter.TryWriteBytes(buffer[1..9], transactionId);
|
||||
BitConverter.TryWriteBytes(buffer[9..17], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
|
||||
_walStream!.Write(buffer);
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes a data modification record
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Writes a data modification record
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="pageId">The page identifier of the modified page.</param>
|
||||
/// <param name="afterImage">The page contents after modification.</param>
|
||||
public void WriteDataRecord(ulong transactionId, uint pageId, ReadOnlySpan<byte> afterImage)
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
WriteDataRecordInternal(transactionId, pageId, afterImage);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes a data modification record asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="transactionId">The transaction identifier.</param>
|
||||
/// <param name="pageId">The page identifier of the modified page.</param>
|
||||
/// <param name="afterImage">The page contents after modification.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous write operation.</returns>
|
||||
public async ValueTask WriteDataRecordAsync(ulong transactionId, uint pageId, ReadOnlyMemory<byte> afterImage, CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
var headerSize = 17;
|
||||
var totalSize = headerSize + afterImage.Length;
|
||||
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(totalSize);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Write;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), transactionId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 4), pageId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(13, 4), afterImage.Length);
|
||||
|
||||
afterImage.Span.CopyTo(buffer.AsSpan(headerSize));
|
||||
|
||||
await _walStream!.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, totalSize), ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteDataRecordInternal(ulong transactionId, uint pageId, ReadOnlySpan<byte> afterImage)
|
||||
{
|
||||
// Header: type(1) + txnId(8) + pageId(4) + afterSize(4) = 17 bytes
|
||||
var headerSize = 17;
|
||||
var totalSize = headerSize + afterImage.Length;
|
||||
|
||||
var buffer = System.Buffers.ArrayPool<byte>.Shared.Rent(totalSize);
|
||||
try
|
||||
{
|
||||
buffer[0] = (byte)WalRecordType.Write;
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(1, 8), transactionId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(9, 4), pageId);
|
||||
BitConverter.TryWriteBytes(buffer.AsSpan(13, 4), afterImage.Length);
|
||||
|
||||
afterImage.CopyTo(buffer.AsSpan(headerSize));
|
||||
|
||||
_walStream!.Write(buffer.AsSpan(0, totalSize));
|
||||
}
|
||||
finally
|
||||
{
|
||||
System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Flushes all buffered writes to disk
|
||||
/// </summary>
|
||||
/// <summary>
|
||||
/// Flushes all buffered writes to disk
|
||||
/// </summary>
|
||||
public void Flush()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
_walStream?.Flush(flushToDisk: true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flushes all buffered writes to disk asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous flush operation.</returns>
|
||||
public async Task FlushAsync(CancellationToken ct = default)
|
||||
{
|
||||
await _lock.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
if (_walStream != null)
|
||||
{
|
||||
await _walStream.FlushAsync(ct);
|
||||
// FlushAsync doesn't guarantee flushToDisk on all platforms/implementations in the same way as Flush(true)
|
||||
// but FileStream in .NET 6+ handles this reasonable well.
|
||||
// For strict durability, we might still want to invoke a sync flush or check platform specifics,
|
||||
// but typically FlushAsync(ct) is sufficient for "Async" pattern.
|
||||
// However, FileStream.FlushAsync() acts like flushToDisk=false by default in older .NET?
|
||||
// Actually, FileStream.Flush() has flushToDisk arg, FlushAsync does not but implementation usually does buffer flush.
|
||||
// To be safe for WAL, we might care about fsync.
|
||||
// For now, just FlushAsync();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current size of the WAL file in bytes
|
||||
/// </summary>
|
||||
public long GetCurrentSize()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
return _walStream?.Length ?? 0;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Truncates the WAL file (removes all content).
|
||||
/// Should only be called after successful checkpoint.
|
||||
/// </summary>
|
||||
public void Truncate()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
if (_walStream != null)
|
||||
{
|
||||
_walStream.SetLength(0);
|
||||
_walStream.Position = 0;
|
||||
_walStream.Flush(flushToDisk: true);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Reads all WAL records (for recovery)
|
||||
/// </summary>
|
||||
public List<WalRecord> ReadAll()
|
||||
{
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
var records = new List<WalRecord>();
|
||||
|
||||
if (_walStream == null || _walStream.Length == 0)
|
||||
return records;
|
||||
|
||||
_walStream.Position = 0;
|
||||
|
||||
// Allocate buffers outside loop to avoid CA2014 warning
|
||||
Span<byte> headerBuf = stackalloc byte[16];
|
||||
Span<byte> dataBuf = stackalloc byte[12];
|
||||
|
||||
while (_walStream.Position < _walStream.Length)
|
||||
{
|
||||
var typeByte = _walStream.ReadByte();
|
||||
if (typeByte == -1) break;
|
||||
|
||||
var type = (WalRecordType)typeByte;
|
||||
|
||||
// Check for invalid record type (file padding or corruption)
|
||||
if (typeByte == 0 || !Enum.IsDefined(typeof(WalRecordType), type))
|
||||
{
|
||||
// Reached end of valid records (file may have padding)
|
||||
break;
|
||||
}
|
||||
|
||||
WalRecord record;
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case WalRecordType.Begin:
|
||||
case WalRecordType.Commit:
|
||||
case WalRecordType.Abort:
|
||||
// Read common fields (txnId + timestamp = 16 bytes)
|
||||
var bytesRead = _walStream.Read(headerBuf);
|
||||
if (bytesRead < 16)
|
||||
{
|
||||
// Incomplete record, stop reading
|
||||
return records;
|
||||
}
|
||||
|
||||
var txnId = BitConverter.ToUInt64(headerBuf[0..8]);
|
||||
var timestamp = BitConverter.ToInt64(headerBuf[8..16]);
|
||||
|
||||
record = new WalRecord
|
||||
{
|
||||
Type = type,
|
||||
TransactionId = txnId,
|
||||
Timestamp = timestamp
|
||||
};
|
||||
break;
|
||||
|
||||
case WalRecordType.Write:
|
||||
// Write records have different format: txnId(8) + pageId(4) + afterSize(4)
|
||||
// Read txnId + pageId + afterSize = 16 bytes
|
||||
bytesRead = _walStream.Read(headerBuf);
|
||||
if (bytesRead < 16)
|
||||
{
|
||||
// Incomplete write record header, stop reading
|
||||
return records;
|
||||
}
|
||||
|
||||
txnId = BitConverter.ToUInt64(headerBuf[0..8]);
|
||||
var pageId = BitConverter.ToUInt32(headerBuf[8..12]);
|
||||
var afterSize = BitConverter.ToInt32(headerBuf[12..16]);
|
||||
|
||||
// Validate afterSize to prevent overflow or corruption
|
||||
if (afterSize < 0 || afterSize > 100 * 1024 * 1024) // Max 100MB per record
|
||||
{
|
||||
// Corrupted size, stop reading
|
||||
return records;
|
||||
}
|
||||
|
||||
var afterImage = new byte[afterSize];
|
||||
|
||||
// Read afterImage
|
||||
if (_walStream.Read(afterImage) < afterSize)
|
||||
{
|
||||
// Incomplete after image, stop reading
|
||||
return records;
|
||||
}
|
||||
|
||||
record = new WalRecord
|
||||
{
|
||||
Type = type,
|
||||
TransactionId = txnId,
|
||||
Timestamp = 0, // Write records don't have timestamp
|
||||
PageId = pageId,
|
||||
AfterImage = afterImage
|
||||
};
|
||||
break;
|
||||
|
||||
default:
|
||||
// Unknown record type, stop reading
|
||||
return records;
|
||||
}
|
||||
|
||||
records.Add(record);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases resources used by the write-ahead log.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
|
||||
_lock.Wait();
|
||||
try
|
||||
{
|
||||
_walStream?.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.Release();
|
||||
_lock.Dispose();
|
||||
}
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a WAL record.
|
||||
/// Implemented as struct for memory efficiency.
|
||||
/// </summary>
|
||||
public struct WalRecord
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the WAL record type.
|
||||
/// </summary>
|
||||
public WalRecordType Type { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the transaction identifier.
|
||||
/// </summary>
|
||||
public ulong TransactionId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the record timestamp in Unix milliseconds.
|
||||
/// </summary>
|
||||
public long Timestamp { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the page identifier for write records.
|
||||
/// </summary>
|
||||
public uint PageId { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the after-image payload for write records.
|
||||
/// </summary>
|
||||
public byte[]? AfterImage { get; set; }
|
||||
}
|
||||
38
src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj
Executable file
38
src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj
Executable file
@@ -0,0 +1,38 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD.Core</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD.Core</RootNamespace>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
|
||||
<PackageId>ZB.MOM.WW.CBDD.Core</PackageId>
|
||||
<Version>1.3.1</Version>
|
||||
<Authors>CBDD Team</Authors>
|
||||
<Description>High-Performance BSON Database Engine Core Library for .NET 10</Description>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
||||
<_Parameter1>ZB.MOM.WW.CBDD.Tests</_Parameter1>
|
||||
</AssemblyAttribute>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
139
src/CBDD.Demo/Program.cs
Executable file
139
src/CBDD.Demo/Program.cs
Executable file
@@ -0,0 +1,139 @@
|
||||
using ZB.MOM.WW.CBDD.Bson;
|
||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||
using System.Collections.Concurrent;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Serilog;
|
||||
using Serilog.Context;
|
||||
|
||||
var serilogLogger = new LoggerConfiguration()
|
||||
.Enrich.FromLogContext()
|
||||
.WriteTo.Console()
|
||||
.CreateLogger();
|
||||
|
||||
using var loggerFactory = LoggerFactory.Create(builder =>
|
||||
{
|
||||
builder.ClearProviders();
|
||||
builder.AddSerilog(serilogLogger, dispose: true);
|
||||
});
|
||||
|
||||
var logger = loggerFactory.CreateLogger("CBDD.Demo");
|
||||
|
||||
// Shared dictionaries for demo
|
||||
var keyMap = new ConcurrentDictionary<string, ushort>(StringComparer.OrdinalIgnoreCase);
|
||||
var keys = new ConcurrentDictionary<ushort, string>();
|
||||
|
||||
// Pre-register some keys
|
||||
ushort nextId = 1;
|
||||
foreach (var k in new[] { "_id", "name", "age", "active", "title", "timestamp", "score" })
|
||||
{
|
||||
keyMap[k] = nextId;
|
||||
keys[nextId] = k;
|
||||
nextId++;
|
||||
}
|
||||
|
||||
// Example 1: Creating and reading BSON documents
|
||||
logger.LogInformation("=== DocumentDb Demo ===");
|
||||
|
||||
// 1. Create a BSON document using builder
|
||||
var document = BsonDocument.Create(keyMap, builder =>
|
||||
{
|
||||
builder
|
||||
.AddObjectId("_id", ObjectId.NewObjectId())
|
||||
.AddString("name", "John Doe")
|
||||
.AddInt32("age", 30)
|
||||
.AddBoolean("active", true);
|
||||
});
|
||||
|
||||
logger.LogInformation("Document created: {DocumentSize} bytes", document.Size);
|
||||
|
||||
// 2. Read fields from document
|
||||
if (document.TryGetString("name", out var name))
|
||||
logger.LogInformation("Name: {Name}", name);
|
||||
|
||||
if (document.TryGetInt32("age", out var age))
|
||||
logger.LogInformation("Age: {Age}", age);
|
||||
|
||||
if (document.TryGetObjectId("_id", out var id))
|
||||
logger.LogInformation("ID: {DocumentId}", id);
|
||||
|
||||
// 3. Manual BSON writing for zero-allocation scenarios
|
||||
Span<byte> buffer = stackalloc byte[512];
|
||||
var writer = new BsonSpanWriter(buffer, keyMap);
|
||||
|
||||
var sizePos = writer.BeginDocument();
|
||||
writer.WriteString("title", "High Performance BSON");
|
||||
writer.WriteInt64("timestamp", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
|
||||
writer.WriteDouble("score", 98.5);
|
||||
writer.EndDocument(sizePos);
|
||||
|
||||
logger.LogInformation("Manual BSON document: {DocumentSize} bytes", writer.Position);
|
||||
|
||||
// 4. Read it back
|
||||
var reader = new BsonSpanReader(buffer[..writer.Position], keys);
|
||||
var docSize = reader.ReadDocumentSize();
|
||||
logger.LogInformation("Reading document of {DocumentSize} bytes...", docSize);
|
||||
|
||||
using var _ = LogContext.PushProperty("Phase", "ReadDocument");
|
||||
while (reader.Remaining > 1)
|
||||
{
|
||||
var type = reader.ReadBsonType();
|
||||
if (type == BsonType.EndOfDocument)
|
||||
break;
|
||||
|
||||
var fieldName = reader.ReadElementHeader();
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case BsonType.String:
|
||||
logger.LogInformation("{Field}: {Value}", fieldName, reader.ReadString());
|
||||
break;
|
||||
case BsonType.Int64:
|
||||
logger.LogInformation("{Field}: {Value}", fieldName, reader.ReadInt64());
|
||||
break;
|
||||
case BsonType.Double:
|
||||
logger.LogInformation("{Field}: {Value}", fieldName, reader.ReadDouble());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Storage engine demo - page-based file
|
||||
var config = PageFileConfig.Default;
|
||||
var pageFile = new PageFile("demo.db", config);
|
||||
|
||||
logger.LogInformation("Opening page file with {PageSize} byte pages...", config.PageSize);
|
||||
pageFile.Open();
|
||||
|
||||
// Allocate a page
|
||||
var pageId = pageFile.AllocatePage();
|
||||
logger.LogInformation("Allocated page ID: {PageId}", pageId);
|
||||
|
||||
// Write a page header
|
||||
Span<byte> pageBuffer = stackalloc byte[config.PageSize];
|
||||
var header = new PageHeader
|
||||
{
|
||||
PageId = pageId,
|
||||
PageType = PageType.Data,
|
||||
FreeBytes = (ushort)(config.PageSize - 32),
|
||||
NextPageId = 0,
|
||||
TransactionId = 1,
|
||||
Checksum = 0
|
||||
};
|
||||
|
||||
header.WriteTo(pageBuffer);
|
||||
pageFile.WritePage(pageId, pageBuffer);
|
||||
logger.LogInformation("Wrote page header (Type: {PageType})", header.PageType);
|
||||
|
||||
// Read it back
|
||||
Span<byte> readBuffer = stackalloc byte[config.PageSize];
|
||||
pageFile.ReadPage(pageId, readBuffer);
|
||||
var readHeader = PageHeader.ReadFrom(readBuffer);
|
||||
|
||||
logger.LogInformation(
|
||||
"Read page header: PageId={PageId}, Type={PageType}, Free={FreeBytes} bytes",
|
||||
readHeader.PageId,
|
||||
readHeader.PageType,
|
||||
readHeader.FreeBytes);
|
||||
|
||||
pageFile.Dispose();
|
||||
|
||||
logger.LogInformation("Demo completed successfully");
|
||||
898
src/CBDD.SourceGenerators/CodeGenerator.cs
Executable file
898
src/CBDD.SourceGenerators/CodeGenerator.cs
Executable file
@@ -0,0 +1,898 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators
|
||||
{
|
||||
public static class CodeGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Generates the mapper class source code for an entity.
|
||||
/// </summary>
|
||||
/// <param name="entity">The entity metadata used for generation.</param>
|
||||
/// <param name="mapperNamespace">The namespace where the mapper class is generated.</param>
|
||||
/// <returns>The generated mapper class source code.</returns>
|
||||
public static string GenerateMapperClass(EntityInfo entity, string mapperNamespace)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var mapperName = GetMapperName(entity.FullTypeName);
|
||||
var keyProp = entity.Properties.FirstOrDefault(p => p.IsKey);
|
||||
var isRoot = entity.IdProperty != null;
|
||||
|
||||
sb.AppendLine("#pragma warning disable CS8604");
|
||||
|
||||
// Class Declaration
|
||||
if (isRoot)
|
||||
{
|
||||
var baseClass = GetBaseMapperClass(keyProp, entity);
|
||||
// Ensure FullTypeName has global:: prefix if not already present (assuming FullTypeName is fully qualified)
|
||||
var entityType = $"global::{entity.FullTypeName}";
|
||||
sb.AppendLine($" public class {mapperName} : global::ZB.MOM.WW.CBDD.Core.Collections.{baseClass}{entityType}>");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" public class {mapperName}");
|
||||
}
|
||||
|
||||
sb.AppendLine($" {{");
|
||||
|
||||
// Converter instance
|
||||
if (keyProp?.ConverterTypeName != null)
|
||||
{
|
||||
sb.AppendLine($" private readonly global::{keyProp.ConverterTypeName} _idConverter = new();");
|
||||
sb.AppendLine();
|
||||
}
|
||||
|
||||
// Generate static setters for private properties (Expression Trees)
|
||||
var privateSetterProps = entity.Properties.Where(p => (!p.HasPublicSetter && p.HasAnySetter) || p.HasInitOnlySetter).ToList();
|
||||
if (privateSetterProps.Any())
|
||||
{
|
||||
sb.AppendLine($" // Cached Expression Tree setters for private properties");
|
||||
foreach (var prop in privateSetterProps)
|
||||
{
|
||||
var entityType = $"global::{entity.FullTypeName}";
|
||||
var propType = QualifyType(prop.TypeName);
|
||||
sb.AppendLine($" private static readonly global::System.Action<{entityType}, {propType}> _setter_{prop.Name} = CreateSetter<{entityType}, {propType}>(\"{prop.Name}\");");
|
||||
}
|
||||
sb.AppendLine();
|
||||
|
||||
sb.AppendLine($" private static global::System.Action<TObj, TVal> CreateSetter<TObj, TVal>(string propertyName)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" var param = global::System.Linq.Expressions.Expression.Parameter(typeof(TObj), \"obj\");");
|
||||
sb.AppendLine($" var value = global::System.Linq.Expressions.Expression.Parameter(typeof(TVal), \"val\");");
|
||||
sb.AppendLine($" var prop = global::System.Linq.Expressions.Expression.Property(param, propertyName);");
|
||||
sb.AppendLine($" var assign = global::System.Linq.Expressions.Expression.Assign(prop, value);");
|
||||
sb.AppendLine($" return global::System.Linq.Expressions.Expression.Lambda<global::System.Action<TObj, TVal>>(assign, param, value).Compile();");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine();
|
||||
}
|
||||
|
||||
// Collection Name (only for root)
|
||||
if (isRoot)
|
||||
{
|
||||
sb.AppendLine($" public override string CollectionName => \"{entity.CollectionName}\";");
|
||||
sb.AppendLine();
|
||||
}
|
||||
else if (entity.Properties.All(p => !p.IsKey))
|
||||
{
|
||||
sb.AppendLine($"// #warning Entity '{entity.Name}' has no defined primary key. Mapper may not support all features.");
|
||||
}
|
||||
|
||||
// Serialize Method
|
||||
GenerateSerializeMethod(sb, entity, isRoot, mapperNamespace);
|
||||
|
||||
sb.AppendLine();
|
||||
|
||||
// Deserialize Method
|
||||
GenerateDeserializeMethod(sb, entity, isRoot, mapperNamespace);
|
||||
|
||||
if (isRoot)
|
||||
{
|
||||
sb.AppendLine();
|
||||
GenerateIdAccessors(sb, entity);
|
||||
}
|
||||
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine("#pragma warning restore CS8604");
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static void GenerateSerializeMethod(StringBuilder sb, EntityInfo entity, bool isRoot, string mapperNamespace)
|
||||
{
|
||||
var entityType = $"global::{entity.FullTypeName}";
|
||||
|
||||
// Always generate SerializeFields (writes only fields, no document wrapper)
|
||||
// This is needed even for root entities, as they may be used as nested objects
|
||||
// Note: BsonSpanWriter is a ref struct, so it must be passed by ref
|
||||
sb.AppendLine($" public void SerializeFields({entityType} entity, ref global::ZB.MOM.WW.CBDD.Bson.BsonSpanWriter writer)");
|
||||
sb.AppendLine($" {{");
|
||||
GenerateFieldWritesCore(sb, entity, mapperNamespace);
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine();
|
||||
|
||||
// Generate Serialize method (with document wrapper)
|
||||
var methodSig = isRoot
|
||||
? $"public override int Serialize({entityType} entity, global::ZB.MOM.WW.CBDD.Bson.BsonSpanWriter writer)"
|
||||
: $"public int Serialize({entityType} entity, global::ZB.MOM.WW.CBDD.Bson.BsonSpanWriter writer)";
|
||||
|
||||
sb.AppendLine($" {methodSig}");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" var startingPos = writer.BeginDocument();");
|
||||
sb.AppendLine();
|
||||
GenerateFieldWritesCore(sb, entity, mapperNamespace);
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" writer.EndDocument(startingPos);");
|
||||
sb.AppendLine($" return writer.Position;");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
|
||||
private static void GenerateFieldWritesCore(StringBuilder sb, EntityInfo entity, string mapperNamespace)
|
||||
{
|
||||
foreach (var prop in entity.Properties)
|
||||
{
|
||||
// Handle key property - serialize as "_id" regardless of property name
|
||||
if (prop.IsKey)
|
||||
{
|
||||
if (prop.ConverterTypeName != null)
|
||||
{
|
||||
var providerProp = new PropertyInfo { TypeName = prop.ProviderTypeName ?? "string" };
|
||||
var idWriteMethod = GetPrimitiveWriteMethod(providerProp, allowKey: true);
|
||||
if (idWriteMethod == "WriteString")
|
||||
{
|
||||
sb.AppendLine($" var convertedId = _idConverter.ConvertToProvider(entity.{prop.Name});");
|
||||
sb.AppendLine($" if (convertedId != null)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" writer.WriteString(\"_id\", convertedId);");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" writer.WriteNull(\"_id\");");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" writer.{idWriteMethod}(\"_id\", _idConverter.ConvertToProvider(entity.{prop.Name}));");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var idWriteMethod = GetPrimitiveWriteMethod(prop, allowKey: true);
|
||||
if (idWriteMethod != null)
|
||||
{
|
||||
sb.AppendLine($" writer.{idWriteMethod}(\"_id\", entity.{prop.Name});");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"#warning Unsupported Id type for '{prop.Name}': {prop.TypeName}. Serialization of '_id' will fail.");
|
||||
sb.AppendLine($" // Unsupported Id type: {prop.TypeName}");
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
GenerateValidation(sb, prop);
|
||||
GenerateWriteProperty(sb, prop, mapperNamespace);
|
||||
}
|
||||
}
|
||||
|
||||
private static void GenerateValidation(StringBuilder sb, PropertyInfo prop)
|
||||
{
|
||||
var isString = prop.TypeName == "string" || prop.TypeName == "String";
|
||||
|
||||
if (prop.IsRequired)
|
||||
{
|
||||
if (isString)
|
||||
{
|
||||
sb.AppendLine($" if (string.IsNullOrEmpty(entity.{prop.Name})) throw new global::System.ComponentModel.DataAnnotations.ValidationException(\"Property {prop.Name} is required.\");");
|
||||
}
|
||||
else if (prop.IsNullable)
|
||||
{
|
||||
sb.AppendLine($" if (entity.{prop.Name} == null) throw new global::System.ComponentModel.DataAnnotations.ValidationException(\"Property {prop.Name} is required.\");");
|
||||
}
|
||||
}
|
||||
|
||||
if (prop.MaxLength.HasValue && isString)
|
||||
{
|
||||
sb.AppendLine($" if ((entity.{prop.Name}?.Length ?? 0) > {prop.MaxLength}) throw new global::System.ComponentModel.DataAnnotations.ValidationException(\"Property {prop.Name} exceeds max length {prop.MaxLength}.\");");
|
||||
}
|
||||
if (prop.MinLength.HasValue && isString)
|
||||
{
|
||||
sb.AppendLine($" if ((entity.{prop.Name}?.Length ?? 0) < {prop.MinLength}) throw new global::System.ComponentModel.DataAnnotations.ValidationException(\"Property {prop.Name} is below min length {prop.MinLength}.\");");
|
||||
}
|
||||
|
||||
if (prop.RangeMin.HasValue || prop.RangeMax.HasValue)
|
||||
{
|
||||
var minStr = prop.RangeMin?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "double.MinValue";
|
||||
var maxStr = prop.RangeMax?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "double.MaxValue";
|
||||
sb.AppendLine($" if ((double)entity.{prop.Name} < {minStr} || (double)entity.{prop.Name} > {maxStr}) throw new global::System.ComponentModel.DataAnnotations.ValidationException(\"Property {prop.Name} is outside range [{minStr}, {maxStr}].\");");
|
||||
}
|
||||
}
|
||||
|
||||
private static void GenerateWriteProperty(StringBuilder sb, PropertyInfo prop, string mapperNamespace)
|
||||
{
|
||||
var fieldName = prop.BsonFieldName;
|
||||
|
||||
if (prop.IsCollection)
|
||||
{
|
||||
// Add null check for nullable collections
|
||||
if (prop.IsNullable)
|
||||
{
|
||||
sb.AppendLine($" if (entity.{prop.Name} != null)");
|
||||
sb.AppendLine($" {{");
|
||||
}
|
||||
|
||||
var arrayVar = $"{prop.Name.ToLower()}Array";
|
||||
var indent = prop.IsNullable ? " " : "";
|
||||
sb.AppendLine($" {indent}var {arrayVar}Pos = writer.BeginArray(\"{fieldName}\");");
|
||||
sb.AppendLine($" {indent}var {prop.Name.ToLower()}Index = 0;");
|
||||
sb.AppendLine($" {indent}foreach (var item in entity.{prop.Name})");
|
||||
sb.AppendLine($" {indent}{{");
|
||||
|
||||
|
||||
if (prop.IsCollectionItemNested)
|
||||
{
|
||||
sb.AppendLine($" {indent} // Nested Object in List");
|
||||
var nestedMapperTypes = GetMapperName(prop.NestedTypeFullName!);
|
||||
sb.AppendLine($" {indent} var {prop.Name.ToLower()}ItemMapper = new global::{mapperNamespace}.{nestedMapperTypes}();");
|
||||
|
||||
sb.AppendLine($" {indent} var itemStartPos = writer.BeginDocument({prop.Name.ToLower()}Index.ToString());");
|
||||
sb.AppendLine($" {indent} {prop.Name.ToLower()}ItemMapper.SerializeFields(item, ref writer);");
|
||||
sb.AppendLine($" {indent} writer.EndDocument(itemStartPos);");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Simplified: pass a dummy PropertyInfo with the item type for primitive collection items
|
||||
var dummyProp = new PropertyInfo { TypeName = prop.CollectionItemType! };
|
||||
var writeMethod = GetPrimitiveWriteMethod(dummyProp);
|
||||
if (writeMethod != null)
|
||||
{
|
||||
sb.AppendLine($" {indent} writer.{writeMethod}({prop.Name.ToLower()}Index.ToString(), item);");
|
||||
}
|
||||
}
|
||||
sb.AppendLine($" {indent} {prop.Name.ToLower()}Index++;");
|
||||
|
||||
sb.AppendLine($" {indent}}}");
|
||||
sb.AppendLine($" {indent}writer.EndArray({arrayVar}Pos);");
|
||||
|
||||
// Close the null check if block
|
||||
if (prop.IsNullable)
|
||||
{
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" writer.WriteNull(\"{fieldName}\");");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
}
|
||||
else if (prop.IsNestedObject)
|
||||
{
|
||||
sb.AppendLine($" if (entity.{prop.Name} != null)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" var {prop.Name.ToLower()}Pos = writer.BeginDocument(\"{fieldName}\");");
|
||||
var nestedMapperType = GetMapperName(prop.NestedTypeFullName!);
|
||||
sb.AppendLine($" var {prop.Name.ToLower()}Mapper = new global::{mapperNamespace}.{nestedMapperType}();");
|
||||
sb.AppendLine($" {prop.Name.ToLower()}Mapper.SerializeFields(entity.{prop.Name}, ref writer);");
|
||||
sb.AppendLine($" writer.EndDocument({prop.Name.ToLower()}Pos);");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" writer.WriteNull(\"{fieldName}\");");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
var writeMethod = GetPrimitiveWriteMethod(prop, allowKey: false);
|
||||
if (writeMethod != null)
|
||||
{
|
||||
if (prop.IsNullable || prop.TypeName == "string" || prop.TypeName == "String")
|
||||
{
|
||||
sb.AppendLine($" if (entity.{prop.Name} != null)");
|
||||
sb.AppendLine($" {{");
|
||||
// For nullable value types, use .Value to unwrap
|
||||
// String is a reference type and doesn't need .Value
|
||||
var isValueTypeNullable = prop.IsNullable && IsValueType(prop.TypeName);
|
||||
var valueAccess = isValueTypeNullable
|
||||
? $"entity.{prop.Name}.Value"
|
||||
: $"entity.{prop.Name}";
|
||||
sb.AppendLine($" writer.{writeMethod}(\"{fieldName}\", {valueAccess});");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" writer.WriteNull(\"{fieldName}\");");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" writer.{writeMethod}(\"{fieldName}\", entity.{prop.Name});");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"#warning Property '{prop.Name}' of type '{prop.TypeName}' is not directly supported and has no converter. It will be skipped during serialization.");
|
||||
sb.AppendLine($" // Unsupported type: {prop.TypeName} for {prop.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void GenerateDeserializeMethod(StringBuilder sb, EntityInfo entity, bool isRoot, string mapperNamespace)
|
||||
{
|
||||
var entityType = $"global::{entity.FullTypeName}";
|
||||
var needsReflection = entity.HasPrivateSetters || entity.HasPrivateOrNoConstructor;
|
||||
|
||||
// Always generate a public Deserialize method that accepts ref (for nested/internal usage)
|
||||
GenerateDeserializeCore(sb, entity, entityType, needsReflection, mapperNamespace);
|
||||
|
||||
// For root entities, also generate the override without ref that calls the ref version
|
||||
if (isRoot)
|
||||
{
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" public override {entityType} Deserialize(global::ZB.MOM.WW.CBDD.Bson.BsonSpanReader reader)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" return Deserialize(ref reader);");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
}
|
||||
|
||||
private static void GenerateDeserializeCore(StringBuilder sb, EntityInfo entity, string entityType, bool needsReflection, string mapperNamespace)
|
||||
{
|
||||
// Public method that always accepts ref for internal/nested usage
|
||||
sb.AppendLine($" public {entityType} Deserialize(ref global::ZB.MOM.WW.CBDD.Bson.BsonSpanReader reader)");
|
||||
sb.AppendLine($" {{");
|
||||
// Use object initializer if possible or constructor, but for now standard new()
|
||||
// To support required properties, we might need a different approach or verify if source generators can detect required.
|
||||
// For now, let's assume standard creation and property setting.
|
||||
// If required properties are present, compiling 'new T()' might fail if they aren't set in initializer.
|
||||
// Alternative: Deserialize into temporary variables then construct.
|
||||
|
||||
// Declare temp variables for all properties
|
||||
foreach (var prop in entity.Properties)
|
||||
{
|
||||
var baseType = QualifyType(prop.TypeName.TrimEnd('?'));
|
||||
|
||||
// Handle collections init
|
||||
if (prop.IsCollection)
|
||||
{
|
||||
var itemType = prop.CollectionItemType;
|
||||
if (prop.IsCollectionItemNested) itemType = $"global::{prop.NestedTypeFullName}"; // Use full name with global::
|
||||
sb.AppendLine($" var {prop.Name.ToLower()} = new global::System.Collections.Generic.List<{itemType}>();");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" {baseType}? {prop.Name.ToLower()} = default;");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Read document size and track boundaries
|
||||
sb.AppendLine($" var docSize = reader.ReadDocumentSize();");
|
||||
sb.AppendLine($" var docEndPos = reader.Position + docSize - 4; // -4 because size includes itself");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" while (reader.Position < docEndPos)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" var bsonType = reader.ReadBsonType();");
|
||||
sb.AppendLine($" if (bsonType == global::ZB.MOM.WW.CBDD.Bson.BsonType.EndOfDocument) break;");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" var elementName = reader.ReadElementHeader();");
|
||||
sb.AppendLine($" switch (elementName)");
|
||||
sb.AppendLine($" {{");
|
||||
|
||||
foreach (var prop in entity.Properties)
|
||||
{
|
||||
var caseName = prop.IsKey ? "_id" : prop.BsonFieldName;
|
||||
sb.AppendLine($" case \"{caseName}\":");
|
||||
|
||||
// Read Logic -> assign to local var
|
||||
GenerateReadPropertyToLocal(sb, prop, "bsonType", mapperNamespace);
|
||||
|
||||
sb.AppendLine($" break;");
|
||||
}
|
||||
|
||||
sb.AppendLine($" default:");
|
||||
sb.AppendLine($" reader.SkipValue(bsonType);");
|
||||
sb.AppendLine($" break;");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine();
|
||||
|
||||
// Construct object - different approach if needs reflection
|
||||
if (needsReflection)
|
||||
{
|
||||
// Use GetUninitializedObject + Expression Trees for private setters
|
||||
sb.AppendLine($" // Creating instance without calling constructor (has private members)");
|
||||
sb.AppendLine($" var entity = (global::{entity.FullTypeName})global::System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(global::{entity.FullTypeName}));");
|
||||
sb.AppendLine();
|
||||
|
||||
// Set properties using setters (Expression Trees for private, direct for public)
|
||||
foreach (var prop in entity.Properties)
|
||||
{
|
||||
var varName = prop.Name.ToLower();
|
||||
var propValue = varName;
|
||||
|
||||
if (prop.IsCollection)
|
||||
{
|
||||
// Convert to appropriate collection type
|
||||
if (prop.IsArray)
|
||||
{
|
||||
propValue += ".ToArray()";
|
||||
}
|
||||
else if (prop.CollectionConcreteTypeName != null)
|
||||
{
|
||||
var concreteType = prop.CollectionConcreteTypeName;
|
||||
var itemType = prop.IsCollectionItemNested ? $"global::{prop.NestedTypeFullName}" : prop.CollectionItemType;
|
||||
|
||||
if (concreteType.Contains("HashSet"))
|
||||
propValue = $"new global::System.Collections.Generic.HashSet<{itemType}>({propValue})";
|
||||
else if (concreteType.Contains("ISet"))
|
||||
propValue = $"new global::System.Collections.Generic.HashSet<{itemType}>({propValue})";
|
||||
else if (concreteType.Contains("LinkedList"))
|
||||
propValue = $"new global::System.Collections.Generic.LinkedList<{itemType}>({propValue})";
|
||||
else if (concreteType.Contains("Queue"))
|
||||
propValue = $"new global::System.Collections.Generic.Queue<{itemType}>({propValue})";
|
||||
else if (concreteType.Contains("Stack"))
|
||||
propValue = $"new global::System.Collections.Generic.Stack<{itemType}>({propValue})";
|
||||
else if (concreteType.Contains("IReadOnlyList") || concreteType.Contains("IReadOnlyCollection"))
|
||||
propValue += ".AsReadOnly()";
|
||||
}
|
||||
}
|
||||
|
||||
// Use appropriate setter
|
||||
if ((!prop.HasPublicSetter && prop.HasAnySetter) || prop.HasInitOnlySetter)
|
||||
{
|
||||
// Use Expression Tree setter (for private or init-only setters)
|
||||
sb.AppendLine($" _setter_{prop.Name}(entity, {propValue} ?? default!);");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Direct property assignment
|
||||
sb.AppendLine($" entity.{prop.Name} = {propValue} ?? default!;");
|
||||
}
|
||||
}
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" return entity;");
|
||||
}
|
||||
else
|
||||
{
|
||||
// Standard object initializer approach
|
||||
sb.AppendLine($" return new {entityType}");
|
||||
sb.AppendLine($" {{");
|
||||
foreach (var prop in entity.Properties)
|
||||
{
|
||||
var val = prop.Name.ToLower();
|
||||
if (prop.IsCollection)
|
||||
{
|
||||
// Convert to appropriate collection type
|
||||
if (prop.IsArray)
|
||||
{
|
||||
val += ".ToArray()";
|
||||
}
|
||||
else if (prop.CollectionConcreteTypeName != null)
|
||||
{
|
||||
var concreteType = prop.CollectionConcreteTypeName;
|
||||
var itemType = prop.IsCollectionItemNested ? $"global::{prop.NestedTypeFullName}" : prop.CollectionItemType;
|
||||
|
||||
// Check if it needs conversion from List
|
||||
if (concreteType.Contains("HashSet"))
|
||||
{
|
||||
val = $"new global::System.Collections.Generic.HashSet<{itemType}>({val})";
|
||||
}
|
||||
else if (concreteType.Contains("ISet"))
|
||||
{
|
||||
val = $"new global::System.Collections.Generic.HashSet<{itemType}>({val})";
|
||||
}
|
||||
else if (concreteType.Contains("LinkedList"))
|
||||
{
|
||||
val = $"new global::System.Collections.Generic.LinkedList<{itemType}>({val})";
|
||||
}
|
||||
else if (concreteType.Contains("Queue"))
|
||||
{
|
||||
val = $"new global::System.Collections.Generic.Queue<{itemType}>({val})";
|
||||
}
|
||||
else if (concreteType.Contains("Stack"))
|
||||
{
|
||||
val = $"new global::System.Collections.Generic.Stack<{itemType}>({val})";
|
||||
}
|
||||
else if (concreteType.Contains("IReadOnlyList") || concreteType.Contains("IReadOnlyCollection"))
|
||||
{
|
||||
val += ".AsReadOnly()";
|
||||
}
|
||||
// Otherwise keep as List (works for List<T>, IList<T>, ICollection<T>, IEnumerable<T>)
|
||||
}
|
||||
}
|
||||
// For nullable properties, don't use ?? default! since null is a valid value
|
||||
if (prop.IsNullable)
|
||||
{
|
||||
sb.AppendLine($" {prop.Name} = {val},");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" {prop.Name} = {val} ?? default!,");
|
||||
}
|
||||
}
|
||||
sb.AppendLine($" }};");
|
||||
}
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
|
||||
private static void GenerateReadPropertyToLocal(StringBuilder sb, PropertyInfo prop, string bsonTypeVar, string mapperNamespace)
|
||||
{
|
||||
var localVar = prop.Name.ToLower();
|
||||
|
||||
if (prop.IsCollection)
|
||||
{
|
||||
var arrVar = prop.Name.ToLower();
|
||||
sb.AppendLine($" // Read Array {prop.Name}");
|
||||
sb.AppendLine($" var {arrVar}ArrSize = reader.ReadDocumentSize();");
|
||||
sb.AppendLine($" var {arrVar}ArrEndPos = reader.Position + {arrVar}ArrSize - 4;");
|
||||
sb.AppendLine($" while (reader.Position < {arrVar}ArrEndPos)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" var itemType = reader.ReadBsonType();");
|
||||
sb.AppendLine($" if (itemType == global::ZB.MOM.WW.CBDD.Bson.BsonType.EndOfDocument) break;");
|
||||
sb.AppendLine($" reader.ReadElementHeader(); // Skip index key");
|
||||
|
||||
if (prop.IsCollectionItemNested)
|
||||
{
|
||||
var nestedMapperTypes = GetMapperName(prop.NestedTypeFullName!);
|
||||
sb.AppendLine($" var {prop.Name.ToLower()}ItemMapper = new global::{mapperNamespace}.{nestedMapperTypes}();");
|
||||
sb.AppendLine($" var item = {prop.Name.ToLower()}ItemMapper.Deserialize(ref reader);");
|
||||
sb.AppendLine($" {localVar}.Add(item);");
|
||||
}
|
||||
else
|
||||
{
|
||||
var readMethod = GetPrimitiveReadMethod(new PropertyInfo { TypeName = prop.CollectionItemType! });
|
||||
if (readMethod != null)
|
||||
{
|
||||
var cast = (prop.CollectionItemType == "float" || prop.CollectionItemType == "Single") ? "(float)" : "";
|
||||
sb.AppendLine($" var item = {cast}reader.{readMethod}();");
|
||||
sb.AppendLine($" {localVar}.Add(item);");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" reader.SkipValue(itemType);");
|
||||
}
|
||||
}
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else if (prop.IsKey && prop.ConverterTypeName != null)
|
||||
{
|
||||
var providerProp = new PropertyInfo { TypeName = prop.ProviderTypeName ?? "string" };
|
||||
var readMethod = GetPrimitiveReadMethod(providerProp);
|
||||
sb.AppendLine($" {localVar} = _idConverter.ConvertFromProvider(reader.{readMethod}());");
|
||||
}
|
||||
else if (prop.IsNestedObject)
|
||||
{
|
||||
sb.AppendLine($" if ({bsonTypeVar} == global::ZB.MOM.WW.CBDD.Bson.BsonType.Null)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" {localVar} = null;");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
var nestedMapperType = GetMapperName(prop.NestedTypeFullName!);
|
||||
sb.AppendLine($" var {prop.Name.ToLower()}Mapper = new global::{mapperNamespace}.{nestedMapperType}();");
|
||||
sb.AppendLine($" {localVar} = {prop.Name.ToLower()}Mapper.Deserialize(ref reader);");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
var readMethod = GetPrimitiveReadMethod(prop);
|
||||
if (readMethod != null)
|
||||
{
|
||||
var cast = (prop.TypeName == "float" || prop.TypeName == "Single") ? "(float)" : "";
|
||||
|
||||
// Handle nullable types - check for null in BSON stream
|
||||
if (prop.IsNullable)
|
||||
{
|
||||
sb.AppendLine($" if ({bsonTypeVar} == global::ZB.MOM.WW.CBDD.Bson.BsonType.Null)");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" {localVar} = null;");
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($" else");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" {localVar} = {cast}reader.{readMethod}();");
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" {localVar} = {cast}reader.{readMethod}();");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" reader.SkipValue({bsonTypeVar});");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a deterministic mapper type name from a fully qualified type name.
|
||||
/// </summary>
|
||||
/// <param name="fullTypeName">The fully qualified entity type name.</param>
|
||||
/// <returns>The generated mapper type name.</returns>
|
||||
public static string GetMapperName(string fullTypeName)
|
||||
{
|
||||
if (string.IsNullOrEmpty(fullTypeName)) return "UnknownMapper";
|
||||
// Remove global:: prefix
|
||||
var cleanName = fullTypeName.Replace("global::", "");
|
||||
// Replace dots, plus (nested classes), and colons (global::) with underscores
|
||||
return cleanName.Replace(".", "_").Replace("+", "_").Replace(":", "_") + "Mapper";
|
||||
}
|
||||
|
||||
private static void GenerateIdAccessors(StringBuilder sb, EntityInfo entity)
|
||||
{
|
||||
var keyProp = entity.Properties.FirstOrDefault(p => p.IsKey);
|
||||
|
||||
// Use CollectionIdTypeFullName if available (from DocumentCollection<TId, T> declaration)
|
||||
string keyType;
|
||||
if (!string.IsNullOrEmpty(entity.CollectionIdTypeFullName))
|
||||
{
|
||||
// Remove "global::" prefix if present
|
||||
keyType = entity.CollectionIdTypeFullName!.Replace("global::", "");
|
||||
}
|
||||
else
|
||||
{
|
||||
keyType = keyProp?.TypeName ?? "ObjectId";
|
||||
}
|
||||
|
||||
// Normalize keyType - remove nullable suffix for the methods
|
||||
// We expect Id to have a value during serialization/deserialization
|
||||
keyType = keyType.TrimEnd('?');
|
||||
|
||||
// Normalize keyType
|
||||
switch (keyType)
|
||||
{
|
||||
case "Int32": keyType = "int"; break;
|
||||
case "Int64": keyType = "long"; break;
|
||||
case "String": keyType = "string"; break;
|
||||
case "Double": keyType = "double"; break;
|
||||
case "Boolean": keyType = "bool"; break;
|
||||
case "Decimal": keyType = "decimal"; break;
|
||||
case "Guid": keyType = "global::System.Guid"; break;
|
||||
case "DateTime": keyType = "global::System.DateTime"; break;
|
||||
case "ObjectId": keyType = "global::ZB.MOM.WW.CBDD.Bson.ObjectId"; break;
|
||||
}
|
||||
|
||||
var entityType = $"global::{entity.FullTypeName}";
|
||||
var qualifiedKeyType = keyType.StartsWith("global::") ? keyType : (keyProp?.ConverterTypeName != null ? $"global::{keyProp.TypeName.TrimEnd('?')}" : keyType);
|
||||
|
||||
var propName = keyProp?.Name ?? "Id";
|
||||
|
||||
// GetId can return nullable if the property is nullable, but we add ! to assert non-null
|
||||
// This helps catch bugs where entities are created without an Id
|
||||
if (keyProp?.IsNullable == true)
|
||||
{
|
||||
sb.AppendLine($" public override {qualifiedKeyType} GetId({entityType} entity) => entity.{propName}!;");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" public override {qualifiedKeyType} GetId({entityType} entity) => entity.{propName};");
|
||||
}
|
||||
|
||||
// If the ID property has a private or init-only setter, use the compiled setter
|
||||
if (entity.HasPrivateSetters && keyProp != null && (!keyProp.HasPublicSetter || keyProp.HasInitOnlySetter))
|
||||
{
|
||||
sb.AppendLine($" public override void SetId({entityType} entity, {qualifiedKeyType} id) => _setter_{propName}(entity, id);");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" public override void SetId({entityType} entity, {qualifiedKeyType} id) => entity.{propName} = id;");
|
||||
}
|
||||
|
||||
if (keyProp?.ConverterTypeName != null)
|
||||
{
|
||||
var providerType = keyProp.ProviderTypeName ?? "string";
|
||||
// Normalize providerType
|
||||
switch (providerType)
|
||||
{
|
||||
case "Int32": providerType = "int"; break;
|
||||
case "Int64": providerType = "long"; break;
|
||||
case "String": providerType = "string"; break;
|
||||
case "Guid": providerType = "global::System.Guid"; break;
|
||||
case "ObjectId": providerType = "global::ZB.MOM.WW.CBDD.Bson.ObjectId"; break;
|
||||
}
|
||||
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" public override global::ZB.MOM.WW.CBDD.Core.Indexing.IndexKey ToIndexKey({qualifiedKeyType} id) => ");
|
||||
sb.AppendLine($" global::ZB.MOM.WW.CBDD.Core.Indexing.IndexKey.Create(_idConverter.ConvertToProvider(id));");
|
||||
sb.AppendLine();
|
||||
sb.AppendLine($" public override {qualifiedKeyType} FromIndexKey(global::ZB.MOM.WW.CBDD.Core.Indexing.IndexKey key) => ");
|
||||
sb.AppendLine($" _idConverter.ConvertFromProvider(key.As<{providerType}>());");
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetBaseMapperClass(PropertyInfo? keyProp, EntityInfo entity)
|
||||
{
|
||||
if (keyProp?.ConverterTypeName != null)
|
||||
{
|
||||
return $"DocumentMapperBase<global::{keyProp.TypeName}, ";
|
||||
}
|
||||
|
||||
// Use CollectionIdTypeFullName if available (from DocumentCollection<TId, T> declaration)
|
||||
string keyType;
|
||||
if (!string.IsNullOrEmpty(entity.CollectionIdTypeFullName))
|
||||
{
|
||||
// Remove "global::" prefix if present
|
||||
keyType = entity.CollectionIdTypeFullName!.Replace("global::", "");
|
||||
}
|
||||
else
|
||||
{
|
||||
keyType = keyProp?.TypeName ?? "ObjectId";
|
||||
}
|
||||
|
||||
// Normalize type by removing nullable suffix (?) for comparison
|
||||
// At serialization time, we expect the Id to always have a value
|
||||
var normalizedKeyType = keyType.TrimEnd('?');
|
||||
|
||||
if (normalizedKeyType.EndsWith("Int32") || normalizedKeyType == "int") return "Int32MapperBase<";
|
||||
if (normalizedKeyType.EndsWith("Int64") || normalizedKeyType == "long") return "Int64MapperBase<";
|
||||
if (normalizedKeyType.EndsWith("String") || normalizedKeyType == "string") return "StringMapperBase<";
|
||||
if (normalizedKeyType.EndsWith("Guid")) return "GuidMapperBase<";
|
||||
if (normalizedKeyType.EndsWith("ObjectId")) return "ObjectIdMapperBase<";
|
||||
|
||||
return "ObjectIdMapperBase<";
|
||||
}
|
||||
|
||||
private static string? GetPrimitiveWriteMethod(PropertyInfo prop, bool allowKey = false)
|
||||
{
|
||||
var typeName = prop.TypeName;
|
||||
if (prop.ColumnTypeName == "point" || prop.ColumnTypeName == "coordinate" || prop.ColumnTypeName == "geopoint")
|
||||
{
|
||||
return "WriteCoordinates";
|
||||
}
|
||||
|
||||
if (typeName.Contains("double") && typeName.Contains(",") && typeName.StartsWith("(") && typeName.EndsWith(")"))
|
||||
{
|
||||
return "WriteCoordinates";
|
||||
}
|
||||
|
||||
var cleanType = typeName.TrimEnd('?').Trim();
|
||||
|
||||
if (cleanType.EndsWith("Int32") || cleanType == "int") return "WriteInt32";
|
||||
if (cleanType.EndsWith("Int64") || cleanType == "long") return "WriteInt64";
|
||||
if (cleanType.EndsWith("String") || cleanType == "string") return "WriteString";
|
||||
if (cleanType.EndsWith("Boolean") || cleanType == "bool") return "WriteBoolean";
|
||||
if (cleanType.EndsWith("Single") || cleanType == "float") return "WriteDouble";
|
||||
if (cleanType.EndsWith("Double") || cleanType == "double") return "WriteDouble";
|
||||
if (cleanType.EndsWith("Decimal") || cleanType == "decimal") return "WriteDecimal128";
|
||||
if (cleanType.EndsWith("DateTime") && !cleanType.EndsWith("DateTimeOffset")) return "WriteDateTime";
|
||||
if (cleanType.EndsWith("DateTimeOffset")) return "WriteDateTimeOffset";
|
||||
if (cleanType.EndsWith("TimeSpan")) return "WriteTimeSpan";
|
||||
if (cleanType.EndsWith("DateOnly")) return "WriteDateOnly";
|
||||
if (cleanType.EndsWith("TimeOnly")) return "WriteTimeOnly";
|
||||
if (cleanType.EndsWith("Guid")) return "WriteGuid";
|
||||
if (cleanType.EndsWith("ObjectId")) return "WriteObjectId";
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? GetPrimitiveReadMethod(PropertyInfo prop)
|
||||
{
|
||||
var typeName = prop.TypeName;
|
||||
if (prop.ColumnTypeName == "point" || prop.ColumnTypeName == "coordinate" || prop.ColumnTypeName == "geopoint")
|
||||
{
|
||||
return "ReadCoordinates";
|
||||
}
|
||||
|
||||
if (typeName.Contains("double") && typeName.Contains(",") && typeName.StartsWith("(") && typeName.EndsWith(")"))
|
||||
{
|
||||
return "ReadCoordinates";
|
||||
}
|
||||
|
||||
var cleanType = typeName.TrimEnd('?').Trim();
|
||||
|
||||
if (cleanType.EndsWith("Int32") || cleanType == "int") return "ReadInt32";
|
||||
if (cleanType.EndsWith("Int64") || cleanType == "long") return "ReadInt64";
|
||||
if (cleanType.EndsWith("String") || cleanType == "string") return "ReadString";
|
||||
if (cleanType.EndsWith("Boolean") || cleanType == "bool") return "ReadBoolean";
|
||||
if (cleanType.EndsWith("Single") || cleanType == "float") return "ReadDouble";
|
||||
if (cleanType.EndsWith("Double") || cleanType == "double") return "ReadDouble";
|
||||
if (cleanType.EndsWith("Decimal") || cleanType == "decimal") return "ReadDecimal128";
|
||||
if (cleanType.EndsWith("DateTime") && !cleanType.EndsWith("DateTimeOffset")) return "ReadDateTime";
|
||||
if (cleanType.EndsWith("DateTimeOffset")) return "ReadDateTimeOffset";
|
||||
if (cleanType.EndsWith("TimeSpan")) return "ReadTimeSpan";
|
||||
if (cleanType.EndsWith("DateOnly")) return "ReadDateOnly";
|
||||
if (cleanType.EndsWith("TimeOnly")) return "ReadTimeOnly";
|
||||
if (cleanType.EndsWith("Guid")) return "ReadGuid";
|
||||
if (cleanType.EndsWith("ObjectId")) return "ReadObjectId";
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static bool IsValueType(string typeName)
|
||||
{
|
||||
// Check if the type is a value type (struct) that requires .Value unwrapping when nullable
|
||||
// String is a reference type and doesn't need .Value
|
||||
var cleanType = typeName.TrimEnd('?').Trim();
|
||||
|
||||
// Common value types
|
||||
if (cleanType.EndsWith("Int32") || cleanType == "int") return true;
|
||||
if (cleanType.EndsWith("Int64") || cleanType == "long") return true;
|
||||
if (cleanType.EndsWith("Boolean") || cleanType == "bool") return true;
|
||||
if (cleanType.EndsWith("Single") || cleanType == "float") return true;
|
||||
if (cleanType.EndsWith("Double") || cleanType == "double") return true;
|
||||
if (cleanType.EndsWith("Decimal") || cleanType == "decimal") return true;
|
||||
if (cleanType.EndsWith("DateTime")) return true;
|
||||
if (cleanType.EndsWith("DateTimeOffset")) return true;
|
||||
if (cleanType.EndsWith("TimeSpan")) return true;
|
||||
if (cleanType.EndsWith("DateOnly")) return true;
|
||||
if (cleanType.EndsWith("TimeOnly")) return true;
|
||||
if (cleanType.EndsWith("Guid")) return true;
|
||||
if (cleanType.EndsWith("ObjectId")) return true;
|
||||
|
||||
// String and other reference types
|
||||
return false;
|
||||
}
|
||||
|
||||
private static string QualifyType(string typeName)
|
||||
{
|
||||
if (string.IsNullOrEmpty(typeName)) return "object";
|
||||
if (typeName.StartsWith("global::")) return typeName;
|
||||
|
||||
var isNullable = typeName.EndsWith("?");
|
||||
var baseType = typeName.TrimEnd('?').Trim();
|
||||
|
||||
if (baseType.StartsWith("(") && baseType.EndsWith(")")) return typeName; // Tuple
|
||||
|
||||
switch (baseType)
|
||||
{
|
||||
case "int":
|
||||
case "long":
|
||||
case "string":
|
||||
case "bool":
|
||||
case "double":
|
||||
case "float":
|
||||
case "decimal":
|
||||
case "byte":
|
||||
case "sbyte":
|
||||
case "short":
|
||||
case "ushort":
|
||||
case "uint":
|
||||
case "ulong":
|
||||
case "char":
|
||||
case "object":
|
||||
case "dynamic":
|
||||
case "void":
|
||||
return baseType + (isNullable ? "?" : "");
|
||||
case "Guid": return "global::System.Guid" + (isNullable ? "?" : "");
|
||||
case "DateTime": return "global::System.DateTime" + (isNullable ? "?" : "");
|
||||
case "DateTimeOffset": return "global::System.DateTimeOffset" + (isNullable ? "?" : "");
|
||||
case "TimeSpan": return "global::System.TimeSpan" + (isNullable ? "?" : "");
|
||||
case "DateOnly": return "global::System.DateOnly" + (isNullable ? "?" : "");
|
||||
case "TimeOnly": return "global::System.TimeOnly" + (isNullable ? "?" : "");
|
||||
case "ObjectId": return "global::ZB.MOM.WW.CBDD.Bson.ObjectId" + (isNullable ? "?" : "");
|
||||
default:
|
||||
return $"global::{typeName}";
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsPrimitive(string typeName)
|
||||
{
|
||||
var cleanType = typeName.TrimEnd('?').Trim();
|
||||
if (cleanType.StartsWith("(") && cleanType.EndsWith(")")) return true;
|
||||
|
||||
switch (cleanType)
|
||||
{
|
||||
case "int":
|
||||
case "long":
|
||||
case "string":
|
||||
case "bool":
|
||||
case "double":
|
||||
case "float":
|
||||
case "decimal":
|
||||
case "byte":
|
||||
case "sbyte":
|
||||
case "short":
|
||||
case "ushort":
|
||||
case "uint":
|
||||
case "ulong":
|
||||
case "char":
|
||||
case "object":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
261
src/CBDD.SourceGenerators/EntityAnalyzer.cs
Executable file
261
src/CBDD.SourceGenerators/EntityAnalyzer.cs
Executable file
@@ -0,0 +1,261 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.CodeAnalysis;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators
|
||||
{
|
||||
public static class EntityAnalyzer
|
||||
{
|
||||
/// <summary>
|
||||
/// Analyzes an entity symbol and builds source-generation metadata.
|
||||
/// </summary>
|
||||
/// <param name="entityType">The entity type symbol to analyze.</param>
|
||||
/// <param name="semanticModel">The semantic model for symbol and syntax analysis.</param>
|
||||
/// <returns>The analyzed entity metadata.</returns>
|
||||
public static EntityInfo Analyze(INamedTypeSymbol entityType, SemanticModel semanticModel)
|
||||
{
|
||||
var entityInfo = new EntityInfo
|
||||
{
|
||||
Name = entityType.Name,
|
||||
Namespace = entityType.ContainingNamespace.ToDisplayString(),
|
||||
FullTypeName = SyntaxHelper.GetFullName(entityType),
|
||||
CollectionName = entityType.Name.ToLowerInvariant() + "s"
|
||||
};
|
||||
|
||||
var tableAttr = AttributeHelper.GetAttribute(entityType, "Table");
|
||||
if (tableAttr != null)
|
||||
{
|
||||
var tableName = tableAttr.ConstructorArguments.Length > 0 ? tableAttr.ConstructorArguments[0].Value?.ToString() : null;
|
||||
var schema = AttributeHelper.GetNamedArgumentValue(tableAttr, "Schema");
|
||||
|
||||
var collectionName = !string.IsNullOrEmpty(tableName) ? tableName! : entityInfo.Name;
|
||||
if (!string.IsNullOrEmpty(schema))
|
||||
{
|
||||
collectionName = $"{schema}.{collectionName}";
|
||||
}
|
||||
entityInfo.CollectionName = collectionName;
|
||||
}
|
||||
|
||||
// Analyze properties of the root entity
|
||||
AnalyzeProperties(entityType, entityInfo.Properties);
|
||||
|
||||
// Check if entity needs reflection-based deserialization
|
||||
// Include properties with private setters or init-only setters (which can't be set outside initializers)
|
||||
entityInfo.HasPrivateSetters = entityInfo.Properties.Any(p => (!p.HasPublicSetter && p.HasAnySetter) || p.HasInitOnlySetter);
|
||||
|
||||
// Check if entity has public parameterless constructor
|
||||
var hasPublicParameterlessConstructor = entityType.Constructors
|
||||
.Any(c => c.DeclaredAccessibility == Accessibility.Public && c.Parameters.Length == 0);
|
||||
entityInfo.HasPrivateOrNoConstructor = !hasPublicParameterlessConstructor;
|
||||
|
||||
// Analyze nested types recursively
|
||||
// We use a dictionary for nested types to ensure uniqueness by name
|
||||
var analyzedTypes = new HashSet<string>();
|
||||
AnalyzeNestedTypesRecursive(entityInfo.Properties, entityInfo.NestedTypes, semanticModel, analyzedTypes, 1, 3);
|
||||
|
||||
// Determine ID property
|
||||
// entityInfo.IdProperty is computed from Properties.FirstOrDefault(p => p.IsKey)
|
||||
|
||||
if (entityInfo.IdProperty == null)
|
||||
{
|
||||
// Fallback to convention: property named "Id"
|
||||
var idProp = entityInfo.Properties.FirstOrDefault(p => p.Name == "Id");
|
||||
if (idProp != null)
|
||||
{
|
||||
idProp.IsKey = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for AutoId (int/long keys)
|
||||
if (entityInfo.IdProperty != null)
|
||||
{
|
||||
var idType = entityInfo.IdProperty.TypeName.TrimEnd('?');
|
||||
if (idType == "int" || idType == "Int32" || idType == "long" || idType == "Int64")
|
||||
{
|
||||
entityInfo.AutoId = true;
|
||||
}
|
||||
}
|
||||
|
||||
return entityInfo;
|
||||
}
|
||||
|
||||
private static void AnalyzeProperties(INamedTypeSymbol typeSymbol, List<PropertyInfo> properties)
|
||||
{
|
||||
// Collect properties from the entire inheritance hierarchy
|
||||
var seenProperties = new HashSet<string>();
|
||||
var currentType = typeSymbol;
|
||||
|
||||
while (currentType != null && currentType.SpecialType != SpecialType.System_Object)
|
||||
{
|
||||
var sourceProps = currentType.GetMembers()
|
||||
.OfType<IPropertySymbol>()
|
||||
.Where(p => p.DeclaredAccessibility == Accessibility.Public && !p.IsStatic);
|
||||
|
||||
foreach (var prop in sourceProps)
|
||||
{
|
||||
// Skip if already seen (overridden property in derived class takes precedence)
|
||||
if (!seenProperties.Add(prop.Name))
|
||||
continue;
|
||||
|
||||
if (AttributeHelper.ShouldIgnore(prop))
|
||||
continue;
|
||||
|
||||
// Skip computed getter-only properties (no setter, no backing field)
|
||||
bool isReadOnlyGetter = prop.SetMethod == null && !SyntaxHelper.HasBackingField(prop);
|
||||
if (isReadOnlyGetter)
|
||||
continue;
|
||||
|
||||
var columnAttr = AttributeHelper.GetAttribute(prop, "Column");
|
||||
var bsonFieldName = AttributeHelper.GetAttributeStringValue(prop, "BsonProperty") ??
|
||||
AttributeHelper.GetAttributeStringValue(prop, "JsonPropertyName");
|
||||
|
||||
if (bsonFieldName == null && columnAttr != null)
|
||||
{
|
||||
bsonFieldName = columnAttr.ConstructorArguments.Length > 0 ? columnAttr.ConstructorArguments[0].Value?.ToString() : null;
|
||||
}
|
||||
|
||||
var propInfo = new PropertyInfo
|
||||
{
|
||||
Name = prop.Name,
|
||||
TypeName = SyntaxHelper.GetTypeName(prop.Type),
|
||||
BsonFieldName = bsonFieldName ?? prop.Name.ToLowerInvariant(),
|
||||
ColumnTypeName = columnAttr != null ? AttributeHelper.GetNamedArgumentValue(columnAttr, "TypeName") : null,
|
||||
IsNullable = SyntaxHelper.IsNullableType(prop.Type),
|
||||
IsKey = AttributeHelper.IsKey(prop),
|
||||
IsRequired = AttributeHelper.HasAttribute(prop, "Required"),
|
||||
|
||||
HasPublicSetter = prop.SetMethod?.DeclaredAccessibility == Accessibility.Public,
|
||||
HasInitOnlySetter = prop.SetMethod?.IsInitOnly == true,
|
||||
HasAnySetter = prop.SetMethod != null,
|
||||
IsReadOnlyGetter = isReadOnlyGetter,
|
||||
BackingFieldName = (prop.SetMethod?.DeclaredAccessibility != Accessibility.Public)
|
||||
? $"<{prop.Name}>k__BackingField"
|
||||
: null
|
||||
};
|
||||
|
||||
// MaxLength / MinLength
|
||||
propInfo.MaxLength = AttributeHelper.GetAttributeIntValue(prop, "MaxLength");
|
||||
propInfo.MinLength = AttributeHelper.GetAttributeIntValue(prop, "MinLength");
|
||||
|
||||
var stringLengthAttr = AttributeHelper.GetAttribute(prop, "StringLength");
|
||||
if (stringLengthAttr != null)
|
||||
{
|
||||
if (stringLengthAttr.ConstructorArguments.Length > 0 && stringLengthAttr.ConstructorArguments[0].Value is int max)
|
||||
propInfo.MaxLength = max;
|
||||
|
||||
var minLenStr = AttributeHelper.GetNamedArgumentValue(stringLengthAttr, "MinimumLength");
|
||||
if (int.TryParse(minLenStr, out var min))
|
||||
propInfo.MinLength = min;
|
||||
}
|
||||
|
||||
// Range
|
||||
var rangeAttr = AttributeHelper.GetAttribute(prop, "Range");
|
||||
if (rangeAttr != null && rangeAttr.ConstructorArguments.Length >= 2)
|
||||
{
|
||||
if (rangeAttr.ConstructorArguments[0].Value is double dmin) propInfo.RangeMin = dmin;
|
||||
else if (rangeAttr.ConstructorArguments[0].Value is int imin) propInfo.RangeMin = (double)imin;
|
||||
|
||||
if (rangeAttr.ConstructorArguments[1].Value is double dmax) propInfo.RangeMax = dmax;
|
||||
else if (rangeAttr.ConstructorArguments[1].Value is int imax) propInfo.RangeMax = (double)imax;
|
||||
}
|
||||
|
||||
if (SyntaxHelper.IsCollectionType(prop.Type, out var itemType))
|
||||
{
|
||||
propInfo.IsCollection = true;
|
||||
propInfo.IsArray = prop.Type is IArrayTypeSymbol;
|
||||
|
||||
// Determine concrete collection type name
|
||||
propInfo.CollectionConcreteTypeName = SyntaxHelper.GetTypeName(prop.Type);
|
||||
|
||||
if (itemType != null)
|
||||
{
|
||||
propInfo.CollectionItemType = SyntaxHelper.GetTypeName(itemType);
|
||||
|
||||
// Check if collection item is nested object
|
||||
if (SyntaxHelper.IsNestedObjectType(itemType))
|
||||
{
|
||||
propInfo.IsCollectionItemNested = true;
|
||||
propInfo.NestedTypeName = itemType.Name;
|
||||
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)itemType);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check for Nested Object
|
||||
else if (SyntaxHelper.IsNestedObjectType(prop.Type))
|
||||
{
|
||||
propInfo.IsNestedObject = true;
|
||||
propInfo.NestedTypeName = prop.Type.Name;
|
||||
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)prop.Type);
|
||||
}
|
||||
|
||||
properties.Add(propInfo);
|
||||
}
|
||||
|
||||
currentType = currentType.BaseType;
|
||||
}
|
||||
}
|
||||
|
||||
private static void AnalyzeNestedTypesRecursive(
|
||||
List<PropertyInfo> properties,
|
||||
Dictionary<string, NestedTypeInfo> targetNestedTypes,
|
||||
SemanticModel semanticModel,
|
||||
HashSet<string> analyzedTypes,
|
||||
int currentDepth,
|
||||
int maxDepth)
|
||||
{
|
||||
if (currentDepth > maxDepth) return;
|
||||
|
||||
// Identify properties that reference nested types (either directly or via collection)
|
||||
var nestedProps = properties
|
||||
.Where(p => (p.IsNestedObject || p.IsCollectionItemNested) && !string.IsNullOrEmpty(p.NestedTypeFullName))
|
||||
.ToList();
|
||||
|
||||
foreach (var prop in nestedProps)
|
||||
{
|
||||
var fullTypeName = prop.NestedTypeFullName!;
|
||||
var simpleName = prop.NestedTypeName!;
|
||||
|
||||
// Avoid cycles
|
||||
if (analyzedTypes.Contains(fullTypeName)) continue;
|
||||
|
||||
// If already in target list, skip
|
||||
if (targetNestedTypes.ContainsKey(fullTypeName)) continue;
|
||||
|
||||
// Try to find the symbol
|
||||
INamedTypeSymbol? nestedTypeSymbol = null;
|
||||
|
||||
// Try by full name
|
||||
nestedTypeSymbol = semanticModel.Compilation.GetTypeByMetadataName(fullTypeName);
|
||||
|
||||
// If not found, try to resolve via semantic model (might be in the same compilation)
|
||||
if (nestedTypeSymbol == null)
|
||||
{
|
||||
// This is more complex, but usually fullTypeName from ToDisplayString() is traceable.
|
||||
// For now, let's assume GetTypeByMetadataName works for fully qualified names.
|
||||
}
|
||||
|
||||
if (nestedTypeSymbol == null) continue;
|
||||
|
||||
analyzedTypes.Add(fullTypeName);
|
||||
|
||||
var nestedInfo = new NestedTypeInfo
|
||||
{
|
||||
Name = simpleName,
|
||||
Namespace = nestedTypeSymbol.ContainingNamespace.ToDisplayString(),
|
||||
FullTypeName = fullTypeName,
|
||||
Depth = currentDepth
|
||||
};
|
||||
|
||||
// Analyze properties of this nested type
|
||||
AnalyzeProperties(nestedTypeSymbol, nestedInfo.Properties);
|
||||
targetNestedTypes[fullTypeName] = nestedInfo;
|
||||
|
||||
// Recurse
|
||||
AnalyzeNestedTypesRecursive(nestedInfo.Properties, nestedInfo.NestedTypes, semanticModel, analyzedTypes, currentDepth + 1, maxDepth);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
121
src/CBDD.SourceGenerators/Helpers/AttributeHelper.cs
Executable file
121
src/CBDD.SourceGenerators/Helpers/AttributeHelper.cs
Executable file
@@ -0,0 +1,121 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Microsoft.CodeAnalysis;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers
|
||||
{
|
||||
public static class AttributeHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Determines whether a property should be ignored during mapping.
|
||||
/// </summary>
|
||||
/// <param name="property">The property symbol to inspect.</param>
|
||||
/// <returns><see langword="true"/> when the property has an ignore attribute; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool ShouldIgnore(IPropertySymbol property)
|
||||
{
|
||||
return HasAttribute(property, "BsonIgnore") ||
|
||||
HasAttribute(property, "JsonIgnore") ||
|
||||
HasAttribute(property, "NotMapped");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a property is marked as a key.
|
||||
/// </summary>
|
||||
/// <param name="property">The property symbol to inspect.</param>
|
||||
/// <returns><see langword="true"/> when the property has a key attribute; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsKey(IPropertySymbol property)
|
||||
{
|
||||
return HasAttribute(property, "Key") ||
|
||||
HasAttribute(property, "BsonId");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the first constructor argument value for the specified attribute as a string.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="attributeName">The attribute name to match.</param>
|
||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
||||
public static string? GetAttributeStringValue(ISymbol symbol, string attributeName)
|
||||
{
|
||||
var attr = GetAttribute(symbol, attributeName);
|
||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
||||
{
|
||||
return attr.ConstructorArguments[0].Value?.ToString();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the first constructor argument value for the specified attribute as an integer.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="attributeName">The attribute name to match.</param>
|
||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
||||
public static int? GetAttributeIntValue(ISymbol symbol, string attributeName)
|
||||
{
|
||||
var attr = GetAttribute(symbol, attributeName);
|
||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
||||
{
|
||||
if (attr.ConstructorArguments[0].Value is int val) return val;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the first constructor argument value for the specified attribute as a double.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="attributeName">The attribute name to match.</param>
|
||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
||||
public static double? GetAttributeDoubleValue(ISymbol symbol, string attributeName)
|
||||
{
|
||||
var attr = GetAttribute(symbol, attributeName);
|
||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
||||
{
|
||||
if (attr.ConstructorArguments[0].Value is double val) return val;
|
||||
if (attr.ConstructorArguments[0].Value is float fval) return (double)fval;
|
||||
if (attr.ConstructorArguments[0].Value is int ival) return (double)ival;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a named argument value from an attribute.
|
||||
/// </summary>
|
||||
/// <param name="attr">The attribute data.</param>
|
||||
/// <param name="name">The named argument key.</param>
|
||||
/// <returns>The named argument value if present; otherwise, <see langword="null"/>.</returns>
|
||||
public static string? GetNamedArgumentValue(AttributeData attr, string name)
|
||||
{
|
||||
return attr.NamedArguments.FirstOrDefault(a => a.Key == name).Value.Value?.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the first attribute that matches the specified name.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="attributeName">The attribute name to match.</param>
|
||||
/// <returns>The matching attribute data if found; otherwise, <see langword="null"/>.</returns>
|
||||
public static AttributeData? GetAttribute(ISymbol symbol, string attributeName)
|
||||
{
|
||||
return symbol.GetAttributes().FirstOrDefault(a =>
|
||||
a.AttributeClass != null &&
|
||||
(a.AttributeClass.Name == attributeName ||
|
||||
a.AttributeClass.Name == attributeName + "Attribute"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a symbol has the specified attribute.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="attributeName">The attribute name to match.</param>
|
||||
/// <returns><see langword="true"/> when a matching attribute exists; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool HasAttribute(ISymbol symbol, string attributeName)
|
||||
{
|
||||
return GetAttribute(symbol, attributeName) != null;
|
||||
}
|
||||
}
|
||||
}
|
||||
253
src/CBDD.SourceGenerators/Helpers/SyntaxHelper.cs
Executable file
253
src/CBDD.SourceGenerators/Helpers/SyntaxHelper.cs
Executable file
@@ -0,0 +1,253 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.CodeAnalysis;
|
||||
using Microsoft.CodeAnalysis.CSharp;
|
||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers
|
||||
{
|
||||
public static class SyntaxHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Determines whether a symbol inherits from a base type with the specified name.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to inspect.</param>
|
||||
/// <param name="baseTypeName">The base type name to match.</param>
|
||||
/// <returns><see langword="true"/> if the symbol inherits from the base type; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool InheritsFrom(INamedTypeSymbol symbol, string baseTypeName)
|
||||
{
|
||||
var current = symbol.BaseType;
|
||||
while (current != null)
|
||||
{
|
||||
if (current.Name == baseTypeName)
|
||||
return true;
|
||||
current = current.BaseType;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds method invocations with a matching method name under the provided syntax node.
|
||||
/// </summary>
|
||||
/// <param name="node">The root syntax node to search.</param>
|
||||
/// <param name="methodName">The method name to match.</param>
|
||||
/// <returns>A list of matching invocation expressions.</returns>
|
||||
public static List<InvocationExpressionSyntax> FindMethodInvocations(SyntaxNode node, string methodName)
|
||||
{
|
||||
return node.DescendantNodes()
|
||||
.OfType<InvocationExpressionSyntax>()
|
||||
.Where(invocation =>
|
||||
{
|
||||
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess)
|
||||
{
|
||||
return memberAccess.Name.Identifier.Text == methodName;
|
||||
}
|
||||
return false;
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the first generic type argument from an invocation, if present.
|
||||
/// </summary>
|
||||
/// <param name="invocation">The invocation to inspect.</param>
|
||||
/// <returns>The generic type argument text, or <see langword="null"/> when not available.</returns>
|
||||
public static string? GetGenericTypeArgument(InvocationExpressionSyntax invocation)
|
||||
{
|
||||
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess &&
|
||||
memberAccess.Name is GenericNameSyntax genericName &&
|
||||
genericName.TypeArgumentList.Arguments.Count > 0)
|
||||
{
|
||||
return genericName.TypeArgumentList.Arguments[0].ToString();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extracts a property name from an expression.
|
||||
/// </summary>
|
||||
/// <param name="expression">The expression to analyze.</param>
|
||||
/// <returns>The property name when resolved; otherwise, <see langword="null"/>.</returns>
|
||||
public static string? GetPropertyName(ExpressionSyntax? expression)
|
||||
{
|
||||
if (expression == null) return null;
|
||||
if (expression is LambdaExpressionSyntax lambda)
|
||||
{
|
||||
return GetPropertyName(lambda.Body as ExpressionSyntax);
|
||||
}
|
||||
if (expression is MemberAccessExpressionSyntax memberAccess)
|
||||
{
|
||||
return memberAccess.Name.Identifier.Text;
|
||||
}
|
||||
if (expression is PrefixUnaryExpressionSyntax prefixUnary && prefixUnary.Operand is MemberAccessExpressionSyntax prefixMember)
|
||||
{
|
||||
return prefixMember.Name.Identifier.Text;
|
||||
}
|
||||
if (expression is PostfixUnaryExpressionSyntax postfixUnary && postfixUnary.Operand is MemberAccessExpressionSyntax postfixMember)
|
||||
{
|
||||
return postfixMember.Name.Identifier.Text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the fully-qualified type name without the global prefix.
|
||||
/// </summary>
|
||||
/// <param name="symbol">The symbol to format.</param>
|
||||
/// <returns>The formatted full type name.</returns>
|
||||
public static string GetFullName(INamedTypeSymbol symbol)
|
||||
{
|
||||
return symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)
|
||||
.Replace("global::", "");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a display name for a type symbol.
|
||||
/// </summary>
|
||||
/// <param name="type">The type symbol to format.</param>
|
||||
/// <returns>The display name.</returns>
|
||||
public static string GetTypeName(ITypeSymbol type)
|
||||
{
|
||||
if (type is INamedTypeSymbol namedType &&
|
||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||
{
|
||||
var underlyingType = namedType.TypeArguments[0];
|
||||
return GetTypeName(underlyingType) + "?";
|
||||
}
|
||||
|
||||
if (type is IArrayTypeSymbol arrayType)
|
||||
{
|
||||
return GetTypeName(arrayType.ElementType) + "[]";
|
||||
}
|
||||
|
||||
if (type is INamedTypeSymbol nt && nt.IsTupleType)
|
||||
{
|
||||
return type.ToDisplayString();
|
||||
}
|
||||
|
||||
return type.ToDisplayString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a type is nullable.
|
||||
/// </summary>
|
||||
/// <param name="type">The type to evaluate.</param>
|
||||
/// <returns><see langword="true"/> if the type is nullable; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsNullableType(ITypeSymbol type)
|
||||
{
|
||||
if (type is INamedTypeSymbol namedType &&
|
||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
return type.NullableAnnotation == NullableAnnotation.Annotated;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a type is a collection and returns its item type when available.
|
||||
/// </summary>
|
||||
/// <param name="type">The type to evaluate.</param>
|
||||
/// <param name="itemType">When this method returns, contains the collection item type if the type is a collection.</param>
|
||||
/// <returns><see langword="true"/> if the type is a collection; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsCollectionType(ITypeSymbol type, out ITypeSymbol? itemType)
|
||||
{
|
||||
itemType = null;
|
||||
|
||||
// Exclude string (it's IEnumerable<char> but not a collection for our purposes)
|
||||
if (type.SpecialType == SpecialType.System_String)
|
||||
return false;
|
||||
|
||||
// Handle arrays
|
||||
if (type is IArrayTypeSymbol arrayType)
|
||||
{
|
||||
itemType = arrayType.ElementType;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check if the type itself is IEnumerable<T>
|
||||
if (type is INamedTypeSymbol namedType && namedType.IsGenericType)
|
||||
{
|
||||
var typeDefName = namedType.OriginalDefinition.ToDisplayString();
|
||||
if (typeDefName == "System.Collections.Generic.IEnumerable<T>" && namedType.TypeArguments.Length == 1)
|
||||
{
|
||||
itemType = namedType.TypeArguments[0];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the type implements IEnumerable<T> by walking all interfaces
|
||||
var enumerableInterface = type.AllInterfaces
|
||||
.FirstOrDefault(i => i.IsGenericType &&
|
||||
i.OriginalDefinition.ToDisplayString() == "System.Collections.Generic.IEnumerable<T>");
|
||||
|
||||
if (enumerableInterface != null && enumerableInterface.TypeArguments.Length == 1)
|
||||
{
|
||||
itemType = enumerableInterface.TypeArguments[0];
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a type should be treated as a primitive value.
|
||||
/// </summary>
|
||||
/// <param name="type">The type to evaluate.</param>
|
||||
/// <returns><see langword="true"/> if the type is primitive-like; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsPrimitiveType(ITypeSymbol type)
|
||||
{
|
||||
if (type is INamedTypeSymbol namedType &&
|
||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||
{
|
||||
type = namedType.TypeArguments[0];
|
||||
}
|
||||
|
||||
if (type.SpecialType != SpecialType.None && type.SpecialType != SpecialType.System_Object)
|
||||
return true;
|
||||
|
||||
var typeName = type.Name;
|
||||
if (typeName == "Guid" || typeName == "DateTime" || typeName == "DateTimeOffset" ||
|
||||
typeName == "TimeSpan" || typeName == "DateOnly" || typeName == "TimeOnly" ||
|
||||
typeName == "Decimal" || typeName == "ObjectId")
|
||||
return true;
|
||||
|
||||
if (type.TypeKind == TypeKind.Enum)
|
||||
return true;
|
||||
|
||||
if (type is INamedTypeSymbol nt && nt.IsTupleType)
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a type should be treated as a nested object.
|
||||
/// </summary>
|
||||
/// <param name="type">The type to evaluate.</param>
|
||||
/// <returns><see langword="true"/> if the type is a nested object; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsNestedObjectType(ITypeSymbol type)
|
||||
{
|
||||
if (IsPrimitiveType(type)) return false;
|
||||
if (type.SpecialType == SpecialType.System_String) return false;
|
||||
if (IsCollectionType(type, out _)) return false;
|
||||
if (type.SpecialType == SpecialType.System_Object) return false;
|
||||
if (type is INamedTypeSymbol nt && nt.IsTupleType) return false;
|
||||
|
||||
return type.TypeKind == TypeKind.Class || type.TypeKind == TypeKind.Struct;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a property has an associated backing field.
|
||||
/// </summary>
|
||||
/// <param name="property">The property to inspect.</param>
|
||||
/// <returns><see langword="true"/> if a backing field is found; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool HasBackingField(IPropertySymbol property)
|
||||
{
|
||||
// Auto-properties have compiler-generated backing fields
|
||||
// Check if there's a field with the pattern <PropertyName>k__BackingField
|
||||
return property.ContainingType.GetMembers()
|
||||
.OfType<IFieldSymbol>()
|
||||
.Any(f => f.AssociatedSymbol?.Equals(property, SymbolEqualityComparer.Default) == true);
|
||||
}
|
||||
}
|
||||
}
|
||||
436
src/CBDD.SourceGenerators/MapperGenerator.cs
Executable file
436
src/CBDD.SourceGenerators/MapperGenerator.cs
Executable file
@@ -0,0 +1,436 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using Microsoft.CodeAnalysis;
|
||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators
|
||||
{
|
||||
public class DbContextInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the simple class name of the DbContext.
|
||||
/// </summary>
|
||||
public string ClassName { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the fully qualified class name of the DbContext.
|
||||
/// </summary>
|
||||
public string FullClassName => string.IsNullOrEmpty(Namespace) ? ClassName : $"{Namespace}.{ClassName}";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the namespace that contains the DbContext.
|
||||
/// </summary>
|
||||
public string Namespace { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source file path where the DbContext was found.
|
||||
/// </summary>
|
||||
public string FilePath { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the DbContext is nested.
|
||||
/// </summary>
|
||||
public bool IsNested { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the DbContext is partial.
|
||||
/// </summary>
|
||||
public bool IsPartial { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the DbContext inherits from another DbContext.
|
||||
/// </summary>
|
||||
public bool HasBaseDbContext { get; set; } // True if inherits from another DbContext (not DocumentDbContext directly)
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the entities discovered for this DbContext.
|
||||
/// </summary>
|
||||
public List<EntityInfo> Entities { get; set; } = new List<EntityInfo>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the collected nested types keyed by full type name.
|
||||
/// </summary>
|
||||
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; set; } = new Dictionary<string, NestedTypeInfo>();
|
||||
}
|
||||
|
||||
[Generator]
|
||||
public class MapperGenerator : IIncrementalGenerator
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes the mapper source generator pipeline.
|
||||
/// </summary>
|
||||
/// <param name="context">The incremental generator initialization context.</param>
|
||||
public void Initialize(IncrementalGeneratorInitializationContext context)
|
||||
{
|
||||
// Find all classes that inherit from DocumentDbContext
|
||||
var dbContextClasses = context.SyntaxProvider
|
||||
.CreateSyntaxProvider(
|
||||
predicate: static (node, _) => IsPotentialDbContext(node),
|
||||
transform: static (ctx, _) => GetDbContextInfo(ctx))
|
||||
.Where(static context => context is not null)
|
||||
.Collect()
|
||||
.SelectMany(static (contexts, _) => contexts.GroupBy(c => c!.FullClassName).Select(g => g.First())!);
|
||||
|
||||
// Generate code for each DbContext
|
||||
context.RegisterSourceOutput(dbContextClasses, static (spc, dbContext) =>
|
||||
{
|
||||
if (dbContext == null) return;
|
||||
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine($"// Found DbContext: {dbContext.ClassName}");
|
||||
sb.AppendLine($"// BaseType: {(dbContext.HasBaseDbContext ? "inherits from another DbContext" : "inherits from DocumentDbContext directly")}");
|
||||
|
||||
|
||||
foreach (var entity in dbContext.Entities)
|
||||
{
|
||||
// Aggregate nested types recursively
|
||||
CollectNestedTypes(entity.NestedTypes, dbContext.GlobalNestedTypes);
|
||||
}
|
||||
|
||||
// Collect namespaces
|
||||
var namespaces = new HashSet<string>
|
||||
{
|
||||
"System",
|
||||
"System.Collections.Generic",
|
||||
"ZB.MOM.WW.CBDD.Bson",
|
||||
"ZB.MOM.WW.CBDD.Core.Collections"
|
||||
};
|
||||
|
||||
// Add Entity namespaces
|
||||
foreach (var entity in dbContext.Entities)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(entity.Namespace))
|
||||
namespaces.Add(entity.Namespace);
|
||||
}
|
||||
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(nested.Namespace))
|
||||
namespaces.Add(nested.Namespace);
|
||||
}
|
||||
|
||||
// Sanitize file path for name uniqueness
|
||||
var safeName = dbContext.ClassName;
|
||||
if (!string.IsNullOrEmpty(dbContext.FilePath))
|
||||
{
|
||||
var fileName = System.IO.Path.GetFileNameWithoutExtension(dbContext.FilePath);
|
||||
safeName += $"_{fileName}";
|
||||
}
|
||||
|
||||
sb.AppendLine("// <auto-generated/>");
|
||||
sb.AppendLine("#nullable enable");
|
||||
foreach (var ns in namespaces.OrderBy(n => n))
|
||||
{
|
||||
sb.AppendLine($"using {ns};");
|
||||
}
|
||||
sb.AppendLine();
|
||||
|
||||
// Use safeName (Context + Filename) to avoid collisions
|
||||
var mapperNamespace = $"{dbContext.Namespace}.{safeName}_Mappers";
|
||||
sb.AppendLine($"namespace {mapperNamespace}");
|
||||
sb.AppendLine($"{{");
|
||||
|
||||
var generatedMappers = new HashSet<string>();
|
||||
|
||||
// Generate Entity Mappers
|
||||
foreach (var entity in dbContext.Entities)
|
||||
{
|
||||
if (generatedMappers.Add(entity.FullTypeName))
|
||||
{
|
||||
sb.AppendLine(CodeGenerator.GenerateMapperClass(entity, mapperNamespace));
|
||||
}
|
||||
}
|
||||
|
||||
// Generate Nested Mappers
|
||||
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
||||
{
|
||||
if (generatedMappers.Add(nested.FullTypeName))
|
||||
{
|
||||
var nestedEntity = new EntityInfo
|
||||
{
|
||||
Name = nested.Name,
|
||||
Namespace = nested.Namespace,
|
||||
FullTypeName = nested.FullTypeName, // Ensure FullTypeName is copied
|
||||
// Helper to copy properties
|
||||
};
|
||||
nestedEntity.Properties.AddRange(nested.Properties);
|
||||
|
||||
sb.AppendLine(CodeGenerator.GenerateMapperClass(nestedEntity, mapperNamespace));
|
||||
}
|
||||
}
|
||||
|
||||
sb.AppendLine($"}}");
|
||||
sb.AppendLine();
|
||||
|
||||
// Partial DbContext for InitializeCollections (Only for top-level partial classes)
|
||||
if (!dbContext.IsNested && dbContext.IsPartial)
|
||||
{
|
||||
sb.AppendLine($"namespace {dbContext.Namespace}");
|
||||
sb.AppendLine($"{{");
|
||||
sb.AppendLine($" public partial class {dbContext.ClassName}");
|
||||
sb.AppendLine($" {{");
|
||||
sb.AppendLine($" protected override void InitializeCollections()");
|
||||
sb.AppendLine($" {{");
|
||||
|
||||
// Call base.InitializeCollections() if this context inherits from another DbContext
|
||||
if (dbContext.HasBaseDbContext)
|
||||
{
|
||||
sb.AppendLine($" base.InitializeCollections();");
|
||||
}
|
||||
|
||||
foreach (var entity in dbContext.Entities)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(entity.CollectionPropertyName))
|
||||
{
|
||||
var mapperName = $"global::{mapperNamespace}.{CodeGenerator.GetMapperName(entity.FullTypeName)}";
|
||||
sb.AppendLine($" this.{entity.CollectionPropertyName} = CreateCollection(new {mapperName}());");
|
||||
}
|
||||
}
|
||||
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine();
|
||||
|
||||
// Generate Set<TId, T>() override
|
||||
var collectionsWithProperties = dbContext.Entities
|
||||
.Where(e => !string.IsNullOrEmpty(e.CollectionPropertyName) && !string.IsNullOrEmpty(e.CollectionIdTypeFullName))
|
||||
.ToList();
|
||||
|
||||
if (collectionsWithProperties.Any())
|
||||
{
|
||||
sb.AppendLine($" public override global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T> Set<TId, T>()");
|
||||
sb.AppendLine($" {{");
|
||||
|
||||
foreach (var entity in collectionsWithProperties)
|
||||
{
|
||||
var entityTypeStr = $"global::{entity.FullTypeName}";
|
||||
var idTypeStr = entity.CollectionIdTypeFullName;
|
||||
sb.AppendLine($" if (typeof(TId) == typeof({idTypeStr}) && typeof(T) == typeof({entityTypeStr}))");
|
||||
sb.AppendLine($" return (global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T>)(object)this.{entity.CollectionPropertyName};");
|
||||
}
|
||||
|
||||
if (dbContext.HasBaseDbContext)
|
||||
{
|
||||
sb.AppendLine($" return base.Set<TId, T>();");
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.AppendLine($" throw new global::System.InvalidOperationException($\"No collection registered for entity type '{{typeof(T).Name}}' with key type '{{typeof(TId).Name}}'.\");");
|
||||
}
|
||||
|
||||
sb.AppendLine($" }}");
|
||||
}
|
||||
|
||||
sb.AppendLine($" }}");
|
||||
sb.AppendLine($"}}");
|
||||
}
|
||||
|
||||
spc.AddSource($"{dbContext.Namespace}.{safeName}.Mappers.g.cs", sb.ToString());
|
||||
});
|
||||
}
|
||||
|
||||
private static void CollectNestedTypes(Dictionary<string, NestedTypeInfo> source, Dictionary<string, NestedTypeInfo> target)
|
||||
{
|
||||
foreach (var kvp in source)
|
||||
{
|
||||
if (!target.ContainsKey(kvp.Value.FullTypeName))
|
||||
{
|
||||
target[kvp.Value.FullTypeName] = kvp.Value;
|
||||
CollectNestedTypes(kvp.Value.NestedTypes, target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void PrintNestedTypes(StringBuilder sb, Dictionary<string, NestedTypeInfo> nestedTypes, string indent)
|
||||
{
|
||||
foreach (var nt in nestedTypes.Values)
|
||||
{
|
||||
sb.AppendLine($"//{indent}- {nt.Name} (Depth: {nt.Depth})");
|
||||
if (nt.Properties.Count > 0)
|
||||
{
|
||||
// Print properties for nested type to be sure
|
||||
foreach (var p in nt.Properties)
|
||||
{
|
||||
var flags = new List<string>();
|
||||
if (p.IsCollection) flags.Add($"Collection<{p.CollectionItemType}>");
|
||||
if (p.IsNestedObject) flags.Add($"Nested<{p.NestedTypeName}>");
|
||||
var flagStr = flags.Any() ? $" [{string.Join(", ", flags)}]" : "";
|
||||
sb.AppendLine($"//{indent} - {p.Name}: {p.TypeName}{flagStr}");
|
||||
}
|
||||
}
|
||||
|
||||
if (nt.NestedTypes.Any())
|
||||
{
|
||||
PrintNestedTypes(sb, nt.NestedTypes, indent + " ");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsPotentialDbContext(SyntaxNode node)
|
||||
{
|
||||
if (node.SyntaxTree.FilePath.EndsWith(".g.cs")) return false;
|
||||
|
||||
return node is ClassDeclarationSyntax classDecl &&
|
||||
classDecl.BaseList != null &&
|
||||
classDecl.Identifier.Text.EndsWith("Context");
|
||||
}
|
||||
|
||||
private static DbContextInfo? GetDbContextInfo(GeneratorSyntaxContext context)
|
||||
{
|
||||
var classDecl = (ClassDeclarationSyntax)context.Node;
|
||||
var semanticModel = context.SemanticModel;
|
||||
|
||||
var classSymbol = semanticModel.GetDeclaredSymbol(classDecl) as INamedTypeSymbol;
|
||||
if (classSymbol == null) return null;
|
||||
|
||||
if (!SyntaxHelper.InheritsFrom(classSymbol, "DocumentDbContext"))
|
||||
return null;
|
||||
|
||||
// Check if this context inherits from another DbContext (not DocumentDbContext directly)
|
||||
var baseType = classSymbol.BaseType;
|
||||
bool hasBaseDbContext = baseType != null &&
|
||||
baseType.Name != "DocumentDbContext" &&
|
||||
SyntaxHelper.InheritsFrom(baseType, "DocumentDbContext");
|
||||
|
||||
var info = new DbContextInfo
|
||||
{
|
||||
ClassName = classSymbol.Name,
|
||||
Namespace = classSymbol.ContainingNamespace.ToDisplayString(),
|
||||
FilePath = classDecl.SyntaxTree.FilePath,
|
||||
IsNested = classSymbol.ContainingType != null,
|
||||
IsPartial = classDecl.Modifiers.Any(m => m.IsKind(Microsoft.CodeAnalysis.CSharp.SyntaxKind.PartialKeyword)),
|
||||
HasBaseDbContext = hasBaseDbContext
|
||||
};
|
||||
|
||||
// Analyze OnModelCreating to find entities
|
||||
var onModelCreating = classDecl.Members
|
||||
.OfType<MethodDeclarationSyntax>()
|
||||
.FirstOrDefault(m => m.Identifier.Text == "OnModelCreating");
|
||||
|
||||
if (onModelCreating != null)
|
||||
{
|
||||
var entityCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "Entity");
|
||||
foreach (var call in entityCalls)
|
||||
{
|
||||
var typeName = SyntaxHelper.GetGenericTypeArgument(call);
|
||||
if (typeName != null)
|
||||
{
|
||||
// Try to find the symbol
|
||||
INamedTypeSymbol? entityType = null;
|
||||
|
||||
// 1. Try by name in current compilation (simple name)
|
||||
var symbols = semanticModel.Compilation.GetSymbolsWithName(typeName);
|
||||
entityType = symbols.OfType<INamedTypeSymbol>().FirstOrDefault();
|
||||
|
||||
// 2. Try by metadata name (if fully qualified)
|
||||
if (entityType == null)
|
||||
{
|
||||
entityType = semanticModel.Compilation.GetTypeByMetadataName(typeName);
|
||||
}
|
||||
|
||||
if (entityType != null)
|
||||
{
|
||||
// Check for duplicates
|
||||
var fullTypeName = SyntaxHelper.GetFullName(entityType);
|
||||
if (!info.Entities.Any(e => e.FullTypeName == fullTypeName))
|
||||
{
|
||||
var entityInfo = EntityAnalyzer.Analyze(entityType, semanticModel);
|
||||
info.Entities.Add(entityInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze OnModelCreating for HasConversion
|
||||
if (onModelCreating != null)
|
||||
{
|
||||
var conversionCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "HasConversion");
|
||||
foreach (var call in conversionCalls)
|
||||
{
|
||||
var converterName = SyntaxHelper.GetGenericTypeArgument(call);
|
||||
if (converterName == null) continue;
|
||||
|
||||
// Trace back: .Property(x => x.Id).HasConversion<T>() or .HasKey(x => x.Id).HasConversion<T>()
|
||||
if (call.Expression is MemberAccessExpressionSyntax { Expression: InvocationExpressionSyntax propertyCall } &&
|
||||
propertyCall.Expression is MemberAccessExpressionSyntax { Name: IdentifierNameSyntax { Identifier: { Text: var propertyMethod } } } &&
|
||||
(propertyMethod == "Property" || propertyMethod == "HasKey"))
|
||||
{
|
||||
var propertyName = SyntaxHelper.GetPropertyName(propertyCall.ArgumentList.Arguments.FirstOrDefault()?.Expression);
|
||||
if (propertyName == null) continue;
|
||||
|
||||
// Trace further back: Entity<T>().Property(...)
|
||||
if (propertyCall.Expression is MemberAccessExpressionSyntax { Expression: InvocationExpressionSyntax entityCall } &&
|
||||
entityCall.Expression is MemberAccessExpressionSyntax { Name: GenericNameSyntax { Identifier: { Text: "Entity" } } })
|
||||
{
|
||||
var entityTypeName = SyntaxHelper.GetGenericTypeArgument(entityCall);
|
||||
if (entityTypeName != null)
|
||||
{
|
||||
var entity = info.Entities.FirstOrDefault(e => e.Name == entityTypeName || e.FullTypeName.EndsWith("." + entityTypeName));
|
||||
if (entity != null)
|
||||
{
|
||||
var prop = entity.Properties.FirstOrDefault(p => p.Name == propertyName);
|
||||
if (prop != null)
|
||||
{
|
||||
// Resolve TProvider from ValueConverter<TModel, TProvider>
|
||||
var converterType = semanticModel.Compilation.GetTypeByMetadataName(converterName) ??
|
||||
semanticModel.Compilation.GetSymbolsWithName(converterName).OfType<INamedTypeSymbol>().FirstOrDefault();
|
||||
|
||||
prop.ConverterTypeName = converterType != null ? SyntaxHelper.GetFullName(converterType) : converterName;
|
||||
|
||||
if (converterType != null && converterType.BaseType != null &&
|
||||
converterType.BaseType.Name == "ValueConverter" &&
|
||||
converterType.BaseType.TypeArguments.Length == 2)
|
||||
{
|
||||
prop.ProviderTypeName = converterType.BaseType.TypeArguments[1].Name;
|
||||
}
|
||||
else if (converterType != null)
|
||||
{
|
||||
// Fallback: search deeper in base types
|
||||
var converterBaseType = converterType.BaseType;
|
||||
while (converterBaseType != null)
|
||||
{
|
||||
if (converterBaseType.Name == "ValueConverter" && converterBaseType.TypeArguments.Length == 2)
|
||||
{
|
||||
prop.ProviderTypeName = converterBaseType.TypeArguments[1].Name;
|
||||
break;
|
||||
}
|
||||
converterBaseType = converterBaseType.BaseType;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze properties to find DocumentCollection<TId, TEntity>
|
||||
var properties = classSymbol.GetMembers().OfType<IPropertySymbol>();
|
||||
foreach (var prop in properties)
|
||||
{
|
||||
if (prop.Type is INamedTypeSymbol namedType &&
|
||||
namedType.OriginalDefinition.Name == "DocumentCollection")
|
||||
{
|
||||
// Expecting 2 type arguments: TId, TEntity
|
||||
if (namedType.TypeArguments.Length == 2)
|
||||
{
|
||||
var entityType = namedType.TypeArguments[1];
|
||||
var entityInfo = info.Entities.FirstOrDefault(e => e.FullTypeName == entityType.ToDisplayString());
|
||||
|
||||
// If found, update
|
||||
if (entityInfo != null)
|
||||
{
|
||||
entityInfo.CollectionPropertyName = prop.Name;
|
||||
entityInfo.CollectionIdTypeFullName = namedType.TypeArguments[0].ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
}
|
||||
}
|
||||
32
src/CBDD.SourceGenerators/Models/DbContextInfo.cs
Executable file
32
src/CBDD.SourceGenerators/Models/DbContextInfo.cs
Executable file
@@ -0,0 +1,32 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models
|
||||
{
|
||||
public class DbContextInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the DbContext class name.
|
||||
/// </summary>
|
||||
public string ClassName { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the namespace containing the DbContext.
|
||||
/// </summary>
|
||||
public string Namespace { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the source file path for the DbContext.
|
||||
/// </summary>
|
||||
public string FilePath { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the entity types discovered for the DbContext.
|
||||
/// </summary>
|
||||
public List<EntityInfo> Entities { get; } = new List<EntityInfo>();
|
||||
|
||||
/// <summary>
|
||||
/// Gets global nested types keyed by type name.
|
||||
/// </summary>
|
||||
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
||||
}
|
||||
}
|
||||
213
src/CBDD.SourceGenerators/Models/EntityInfo.cs
Executable file
213
src/CBDD.SourceGenerators/Models/EntityInfo.cs
Executable file
@@ -0,0 +1,213 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models
|
||||
{
|
||||
/// <summary>
|
||||
/// Contains metadata describing an entity discovered by source generation.
|
||||
/// </summary>
|
||||
public class EntityInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the entity name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the entity namespace.
|
||||
/// </summary>
|
||||
public string Namespace { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the fully qualified entity type name.
|
||||
/// </summary>
|
||||
public string FullTypeName { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the collection name for the entity.
|
||||
/// </summary>
|
||||
public string CollectionName { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the collection property name.
|
||||
/// </summary>
|
||||
public string? CollectionPropertyName { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the fully qualified collection identifier type name.
|
||||
/// </summary>
|
||||
public string? CollectionIdTypeFullName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the key property for the entity if one exists.
|
||||
/// </summary>
|
||||
public PropertyInfo? IdProperty => Properties.FirstOrDefault(p => p.IsKey);
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether IDs are automatically generated.
|
||||
/// </summary>
|
||||
public bool AutoId { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the entity uses private setters.
|
||||
/// </summary>
|
||||
public bool HasPrivateSetters { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the entity has a private or missing constructor.
|
||||
/// </summary>
|
||||
public bool HasPrivateOrNoConstructor { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the entity properties.
|
||||
/// </summary>
|
||||
public List<PropertyInfo> Properties { get; } = new List<PropertyInfo>();
|
||||
/// <summary>
|
||||
/// Gets nested type metadata keyed by type name.
|
||||
/// </summary>
|
||||
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
||||
/// <summary>
|
||||
/// Gets property names that should be ignored by mapping.
|
||||
/// </summary>
|
||||
public HashSet<string> IgnoredProperties { get; } = new HashSet<string>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Contains metadata describing a mapped property.
|
||||
/// </summary>
|
||||
public class PropertyInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the property name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the property type name.
|
||||
/// </summary>
|
||||
public string TypeName { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the BSON field name.
|
||||
/// </summary>
|
||||
public string BsonFieldName { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the database column type name.
|
||||
/// </summary>
|
||||
public string? ColumnTypeName { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is nullable.
|
||||
/// </summary>
|
||||
public bool IsNullable { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property has a public setter.
|
||||
/// </summary>
|
||||
public bool HasPublicSetter { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property uses an init-only setter.
|
||||
/// </summary>
|
||||
public bool HasInitOnlySetter { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property has any setter.
|
||||
/// </summary>
|
||||
public bool HasAnySetter { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the getter is read-only.
|
||||
/// </summary>
|
||||
public bool IsReadOnlyGetter { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the backing field name if available.
|
||||
/// </summary>
|
||||
public string? BackingFieldName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is the key.
|
||||
/// </summary>
|
||||
public bool IsKey { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is required.
|
||||
/// </summary>
|
||||
public bool IsRequired { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum allowed length.
|
||||
/// </summary>
|
||||
public int? MaxLength { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the minimum allowed length.
|
||||
/// </summary>
|
||||
public int? MinLength { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the minimum allowed range value.
|
||||
/// </summary>
|
||||
public double? RangeMin { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the maximum allowed range value.
|
||||
/// </summary>
|
||||
public double? RangeMax { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is a collection.
|
||||
/// </summary>
|
||||
public bool IsCollection { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is an array.
|
||||
/// </summary>
|
||||
public bool IsArray { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the collection item type name.
|
||||
/// </summary>
|
||||
public string? CollectionItemType { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the concrete collection type name.
|
||||
/// </summary>
|
||||
public string? CollectionConcreteTypeName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether the property is a nested object.
|
||||
/// </summary>
|
||||
public bool IsNestedObject { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether collection items are nested objects.
|
||||
/// </summary>
|
||||
public bool IsCollectionItemNested { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the nested type name.
|
||||
/// </summary>
|
||||
public string? NestedTypeName { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the fully qualified nested type name.
|
||||
/// </summary>
|
||||
public string? NestedTypeFullName { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the converter type name.
|
||||
/// </summary>
|
||||
public string? ConverterTypeName { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the provider type name used by the converter.
|
||||
/// </summary>
|
||||
public string? ProviderTypeName { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Contains metadata describing a nested type.
|
||||
/// </summary>
|
||||
public class NestedTypeInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the nested type name.
|
||||
/// </summary>
|
||||
public string Name { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the nested type namespace.
|
||||
/// </summary>
|
||||
public string Namespace { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the fully qualified nested type name.
|
||||
/// </summary>
|
||||
public string FullTypeName { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Gets or sets the depth of the nested type.
|
||||
/// </summary>
|
||||
public int Depth { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the nested type properties.
|
||||
/// </summary>
|
||||
public List<PropertyInfo> Properties { get; } = new List<PropertyInfo>();
|
||||
/// <summary>
|
||||
/// Gets nested type metadata keyed by type name.
|
||||
/// </summary>
|
||||
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
||||
}
|
||||
}
|
||||
40
src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj
Executable file
40
src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj
Executable file
@@ -0,0 +1,40 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
|
||||
<IsRoslynComponent>true</IsRoslynComponent>
|
||||
<PackageId>ZB.MOM.WW.CBDD.SourceGenerators</PackageId>
|
||||
<Version>1.3.1</Version>
|
||||
<Authors>CBDD Team</Authors>
|
||||
<Description>Source Generators for CBDD High-Performance BSON Database Engine</Description>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation;source-generator</PackageTags>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<IncludeBuildOutput>false</IncludeBuildOutput>
|
||||
<DevelopmentDependency>true</DevelopmentDependency>
|
||||
<NoPackageAnalysis>true</NoPackageAnalysis>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" PrivateAssets="all" />
|
||||
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" PrivateAssets="all" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
|
||||
|
||||
<None Include="_._" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
0
src/CBDD.SourceGenerators/_._
Executable file
0
src/CBDD.SourceGenerators/_._
Executable file
34
src/CBDD/ZB.MOM.WW.CBDD.csproj
Executable file
34
src/CBDD/ZB.MOM.WW.CBDD.csproj
Executable file
@@ -0,0 +1,34 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<AssemblyName>ZB.MOM.WW.CBDD</AssemblyName>
|
||||
<RootNamespace>ZB.MOM.WW.CBDD</RootNamespace>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
|
||||
<PackageId>ZB.MOM.WW.CBDD</PackageId>
|
||||
<Version>1.3.1</Version>
|
||||
<Authors>CBDD Team</Authors>
|
||||
<Description>High-Performance BSON Database Engine for .NET 10</Description>
|
||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\CBDD.Core\ZB.MOM.WW.CBDD.Core.csproj" />
|
||||
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj" />
|
||||
<ProjectReference Include="..\CBDD.SourceGenerators\ZB.MOM.WW.CBDD.SourceGenerators.csproj" PrivateAssets="none" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
Reference in New Issue
Block a user