Reformat / cleanup
This commit is contained in:
22
CBDD.slnx
22
CBDD.slnx
@@ -1,12 +1,12 @@
|
|||||||
<Solution>
|
<Solution>
|
||||||
<Folder Name="/src/">
|
<Folder Name="/src/">
|
||||||
<Project Path="src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj" />
|
<Project Path="src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj"/>
|
||||||
<Project Path="src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj" />
|
<Project Path="src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj"/>
|
||||||
<Project Path="src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj" />
|
<Project Path="src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj"/>
|
||||||
<Project Path="src/CBDD/ZB.MOM.WW.CBDD.csproj" />
|
<Project Path="src/CBDD/ZB.MOM.WW.CBDD.csproj"/>
|
||||||
</Folder>
|
</Folder>
|
||||||
<Folder Name="/tests/">
|
<Folder Name="/tests/">
|
||||||
<Project Path="tests/CBDD.Tests.Benchmark/ZB.MOM.WW.CBDD.Tests.Benchmark.csproj" />
|
<Project Path="tests/CBDD.Tests.Benchmark/ZB.MOM.WW.CBDD.Tests.Benchmark.csproj"/>
|
||||||
<Project Path="tests/CBDD.Tests/ZB.MOM.WW.CBDD.Tests.csproj" />
|
<Project Path="tests/CBDD.Tests/ZB.MOM.WW.CBDD.Tests.csproj"/>
|
||||||
</Folder>
|
</Folder>
|
||||||
</Solution>
|
</Solution>
|
||||||
|
|||||||
34
README.md
34
README.md
@@ -1,16 +1,21 @@
|
|||||||
# CBDD
|
# CBDD
|
||||||
|
|
||||||
CBDD is an embedded, document-oriented database engine for .NET 10. It targets internal platform teams that need predictable ACID behavior, low-latency local persistence, and typed access patterns without running an external database server.
|
CBDD is an embedded, document-oriented database engine for .NET 10. It targets internal platform teams that need
|
||||||
|
predictable ACID behavior, low-latency local persistence, and typed access patterns without running an external database
|
||||||
|
server.
|
||||||
|
|
||||||
## Purpose And Business Context
|
## Purpose And Business Context
|
||||||
|
|
||||||
CBDD provides a local data layer for services and tools that need transactional durability, deterministic startup, and high-throughput reads/writes. The primary business outcome is reducing operational overhead for workloads that do not require a networked database cluster.
|
CBDD provides a local data layer for services and tools that need transactional durability, deterministic startup, and
|
||||||
|
high-throughput reads/writes. The primary business outcome is reducing operational overhead for workloads that do not
|
||||||
|
require a networked database cluster.
|
||||||
|
|
||||||
## Ownership And Support
|
## Ownership And Support
|
||||||
|
|
||||||
- Owning team: CBDD maintainers (repository owner: `@dohertj2`)
|
- Owning team: CBDD maintainers (repository owner: `@dohertj2`)
|
||||||
- Primary support path: open a Gitea issue in this repository with labels `incident` or `bug`
|
- Primary support path: open a Gitea issue in this repository with labels `incident` or `bug`
|
||||||
- Escalation path: follow [`docs/runbook.md`](docs/runbook.md) and page the release maintainer listed in the active release PR
|
- Escalation path: follow [`docs/runbook.md`](docs/runbook.md) and page the release maintainer listed in the active
|
||||||
|
release PR
|
||||||
|
|
||||||
## Architecture Overview
|
## Architecture Overview
|
||||||
|
|
||||||
@@ -22,6 +27,7 @@ CBDD has four primary layers:
|
|||||||
4. Source-generated mapping (`src/CBDD.SourceGenerators`)
|
4. Source-generated mapping (`src/CBDD.SourceGenerators`)
|
||||||
|
|
||||||
Detailed architecture material:
|
Detailed architecture material:
|
||||||
|
|
||||||
- [`docs/architecture.md`](docs/architecture.md)
|
- [`docs/architecture.md`](docs/architecture.md)
|
||||||
- [`RFC.md`](RFC.md)
|
- [`RFC.md`](RFC.md)
|
||||||
- [`C-BSON.md`](C-BSON.md)
|
- [`C-BSON.md`](C-BSON.md)
|
||||||
@@ -36,34 +42,44 @@ Detailed architecture material:
|
|||||||
## Setup And Local Run
|
## Setup And Local Run
|
||||||
|
|
||||||
1. Clone the repository.
|
1. Clone the repository.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git clone https://gitea.dohertylan.com/dohertj2/CBDD.git
|
git clone https://gitea.dohertylan.com/dohertj2/CBDD.git
|
||||||
cd CBDD
|
cd CBDD
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected outcome: local repository checkout with `CBDD.slnx` present.
|
Expected outcome: local repository checkout with `CBDD.slnx` present.
|
||||||
|
|
||||||
2. Restore dependencies.
|
2. Restore dependencies.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dotnet restore
|
dotnet restore
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected outcome: restore completes without package errors.
|
Expected outcome: restore completes without package errors.
|
||||||
|
|
||||||
3. Build the solution.
|
3. Build the solution.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dotnet build CBDD.slnx -c Release
|
dotnet build CBDD.slnx -c Release
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected outcome: solution builds without compiler errors.
|
Expected outcome: solution builds without compiler errors.
|
||||||
|
|
||||||
4. Run tests.
|
4. Run tests.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dotnet test CBDD.slnx -c Release
|
dotnet test CBDD.slnx -c Release
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected outcome: all tests pass.
|
Expected outcome: all tests pass.
|
||||||
|
|
||||||
5. Run the full repository fitness check.
|
5. Run the full repository fitness check.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bash scripts/fitness-check.sh
|
bash scripts/fitness-check.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
Expected outcome: format, build, tests, coverage threshold, and package checks complete.
|
Expected outcome: format, build, tests, coverage threshold, and package checks complete.
|
||||||
|
|
||||||
## Configuration And Secrets
|
## Configuration And Secrets
|
||||||
@@ -135,9 +151,12 @@ if (!result.Executed)
|
|||||||
|
|
||||||
Common issues and remediation:
|
Common issues and remediation:
|
||||||
|
|
||||||
- Build/test environment failures: [`docs/troubleshooting.md#build-and-test-failures`](docs/troubleshooting.md#build-and-test-failures)
|
- Build/test environment failures: [
|
||||||
- Data-file recovery procedures: [`docs/troubleshooting.md#data-file-and-recovery-issues`](docs/troubleshooting.md#data-file-and-recovery-issues)
|
`docs/troubleshooting.md#build-and-test-failures`](docs/troubleshooting.md#build-and-test-failures)
|
||||||
- Query/index behavior verification: [`docs/troubleshooting.md#query-and-index-issues`](docs/troubleshooting.md#query-and-index-issues)
|
- Data-file recovery procedures: [
|
||||||
|
`docs/troubleshooting.md#data-file-and-recovery-issues`](docs/troubleshooting.md#data-file-and-recovery-issues)
|
||||||
|
- Query/index behavior verification: [
|
||||||
|
`docs/troubleshooting.md#query-and-index-issues`](docs/troubleshooting.md#query-and-index-issues)
|
||||||
|
|
||||||
## Change Governance
|
## Change Governance
|
||||||
|
|
||||||
@@ -150,4 +169,5 @@ Common issues and remediation:
|
|||||||
|
|
||||||
- Documentation home: [`docs/README.md`](docs/README.md)
|
- Documentation home: [`docs/README.md`](docs/README.md)
|
||||||
- Major feature inventory: [`docs/features/README.md`](docs/features/README.md)
|
- Major feature inventory: [`docs/features/README.md`](docs/features/README.md)
|
||||||
- Architecture decisions: [`docs/adr/0001-storage-engine-and-source-generation.md`](docs/adr/0001-storage-engine-and-source-generation.md)
|
- Architecture decisions: [
|
||||||
|
`docs/adr/0001-storage-engine-and-source-generation.md`](docs/adr/0001-storage-engine-and-source-generation.md)
|
||||||
|
|||||||
@@ -1,60 +1,64 @@
|
|||||||
using System;
|
using System.Collections.Concurrent;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents an in-memory BSON document with lazy parsing.
|
/// Represents an in-memory BSON document with lazy parsing.
|
||||||
/// Uses Memory<byte> to store raw BSON data for zero-copy operations.
|
/// Uses Memory<byte> to store raw BSON data for zero-copy operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class BsonDocument
|
public sealed class BsonDocument
|
||||||
{
|
{
|
||||||
|
private readonly ConcurrentDictionary<ushort, string>? _keys;
|
||||||
private readonly Memory<byte> _rawData;
|
private readonly Memory<byte> _rawData;
|
||||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? _keys;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BsonDocument"/> class from raw BSON memory.
|
/// Initializes a new instance of the <see cref="BsonDocument" /> class from raw BSON memory.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rawBsonData">The raw BSON data.</param>
|
/// <param name="rawBsonData">The raw BSON data.</param>
|
||||||
/// <param name="keys">The optional key dictionary.</param>
|
/// <param name="keys">The optional key dictionary.</param>
|
||||||
public BsonDocument(Memory<byte> rawBsonData, System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? keys = null)
|
public BsonDocument(Memory<byte> rawBsonData, ConcurrentDictionary<ushort, string>? keys = null)
|
||||||
{
|
{
|
||||||
_rawData = rawBsonData;
|
_rawData = rawBsonData;
|
||||||
_keys = keys;
|
_keys = keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BsonDocument"/> class from raw BSON bytes.
|
/// Initializes a new instance of the <see cref="BsonDocument" /> class from raw BSON bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rawBsonData">The raw BSON data.</param>
|
/// <param name="rawBsonData">The raw BSON data.</param>
|
||||||
/// <param name="keys">The optional key dictionary.</param>
|
/// <param name="keys">The optional key dictionary.</param>
|
||||||
public BsonDocument(byte[] rawBsonData, System.Collections.Concurrent.ConcurrentDictionary<ushort, string>? keys = null)
|
public BsonDocument(byte[] rawBsonData, ConcurrentDictionary<ushort, string>? keys = null)
|
||||||
{
|
{
|
||||||
_rawData = rawBsonData;
|
_rawData = rawBsonData;
|
||||||
_keys = keys;
|
_keys = keys;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the raw BSON bytes
|
|
||||||
/// </summary>
|
|
||||||
public ReadOnlySpan<byte> RawData => _rawData.Span;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the document size in bytes
|
|
||||||
/// </summary>
|
|
||||||
public int Size => BitConverter.ToInt32(_rawData.Span[..4]);
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Creates a reader for this document
|
|
||||||
/// </summary>
|
|
||||||
public BsonSpanReader GetReader() => new BsonSpanReader(_rawData.Span, _keys ?? new System.Collections.Concurrent.ConcurrentDictionary<ushort, string>());
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Tries to get a field value by name.
|
/// Gets the raw BSON bytes
|
||||||
/// Returns false if field not found.
|
/// </summary>
|
||||||
|
public ReadOnlySpan<byte> RawData => _rawData.Span;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the document size in bytes
|
||||||
|
/// </summary>
|
||||||
|
public int Size => BitConverter.ToInt32(_rawData.Span[..4]);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a reader for this document
|
||||||
|
/// </summary>
|
||||||
|
public BsonSpanReader GetReader()
|
||||||
|
{
|
||||||
|
return new BsonSpanReader(_rawData.Span,
|
||||||
|
_keys ?? new ConcurrentDictionary<ushort, string>());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Tries to get a field value by name.
|
||||||
|
/// Returns false if field not found.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fieldName">The field name.</param>
|
/// <param name="fieldName">The field name.</param>
|
||||||
/// <param name="value">When this method returns, contains the field value if found; otherwise <see langword="null"/>.</param>
|
/// <param name="value">When this method returns, contains the field value if found; otherwise <see langword="null" />.</param>
|
||||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the field is found; otherwise, <see langword="false" />.</returns>
|
||||||
public bool TryGetString(string fieldName, out string? value)
|
public bool TryGetString(string fieldName, out string? value)
|
||||||
{
|
{
|
||||||
value = null;
|
value = null;
|
||||||
@@ -66,30 +70,30 @@ public sealed class BsonDocument
|
|||||||
fieldName = fieldName.ToLowerInvariant();
|
fieldName = fieldName.ToLowerInvariant();
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var type = reader.ReadBsonType();
|
var type = reader.ReadBsonType();
|
||||||
if (type == BsonType.EndOfDocument)
|
if (type == BsonType.EndOfDocument)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var name = reader.ReadElementHeader();
|
string name = reader.ReadElementHeader();
|
||||||
|
|
||||||
|
if (name == fieldName && type == BsonType.String)
|
||||||
|
{
|
||||||
|
value = reader.ReadString();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.SkipValue(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (name == fieldName && type == BsonType.String)
|
|
||||||
{
|
|
||||||
value = reader.ReadString();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
reader.SkipValue(type);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Tries to get an Int32 field value by name.
|
/// Tries to get an Int32 field value by name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fieldName">The field name.</param>
|
/// <param name="fieldName">The field name.</param>
|
||||||
/// <param name="value">When this method returns, contains the field value if found; otherwise zero.</param>
|
/// <param name="value">When this method returns, contains the field value if found; otherwise zero.</param>
|
||||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the field is found; otherwise, <see langword="false" />.</returns>
|
||||||
public bool TryGetInt32(string fieldName, out int value)
|
public bool TryGetInt32(string fieldName, out int value)
|
||||||
{
|
{
|
||||||
value = 0;
|
value = 0;
|
||||||
@@ -101,30 +105,30 @@ public sealed class BsonDocument
|
|||||||
fieldName = fieldName.ToLowerInvariant();
|
fieldName = fieldName.ToLowerInvariant();
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var type = reader.ReadBsonType();
|
var type = reader.ReadBsonType();
|
||||||
if (type == BsonType.EndOfDocument)
|
if (type == BsonType.EndOfDocument)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var name = reader.ReadElementHeader();
|
string name = reader.ReadElementHeader();
|
||||||
|
|
||||||
|
if (name == fieldName && type == BsonType.Int32)
|
||||||
|
{
|
||||||
|
value = reader.ReadInt32();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.SkipValue(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (name == fieldName && type == BsonType.Int32)
|
|
||||||
{
|
|
||||||
value = reader.ReadInt32();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
reader.SkipValue(type);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Tries to get an ObjectId field value by name.
|
/// Tries to get an ObjectId field value by name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fieldName">The field name.</param>
|
/// <param name="fieldName">The field name.</param>
|
||||||
/// <param name="value">When this method returns, contains the field value if found; otherwise default.</param>
|
/// <param name="value">When this method returns, contains the field value if found; otherwise default.</param>
|
||||||
/// <returns><see langword="true"/> if the field is found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the field is found; otherwise, <see langword="false" />.</returns>
|
||||||
public bool TryGetObjectId(string fieldName, out ObjectId value)
|
public bool TryGetObjectId(string fieldName, out ObjectId value)
|
||||||
{
|
{
|
||||||
value = default;
|
value = default;
|
||||||
@@ -136,52 +140,53 @@ public sealed class BsonDocument
|
|||||||
fieldName = fieldName.ToLowerInvariant();
|
fieldName = fieldName.ToLowerInvariant();
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var type = reader.ReadBsonType();
|
var type = reader.ReadBsonType();
|
||||||
if (type == BsonType.EndOfDocument)
|
if (type == BsonType.EndOfDocument)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var name = reader.ReadElementHeader();
|
string name = reader.ReadElementHeader();
|
||||||
|
|
||||||
|
if (name == fieldName && type == BsonType.ObjectId)
|
||||||
|
{
|
||||||
|
value = reader.ReadObjectId();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.SkipValue(type);
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (name == fieldName && type == BsonType.ObjectId)
|
|
||||||
{
|
|
||||||
value = reader.ReadObjectId();
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
reader.SkipValue(type);
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new BsonDocument from field values using a builder pattern
|
/// Creates a new BsonDocument from field values using a builder pattern
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keyMap">The key map used for field name encoding.</param>
|
/// <param name="keyMap">The key map used for field name encoding.</param>
|
||||||
/// <param name="buildAction">The action that populates the builder.</param>
|
/// <param name="buildAction">The action that populates the builder.</param>
|
||||||
/// <returns>The created BSON document.</returns>
|
/// <returns>The created BSON document.</returns>
|
||||||
public static BsonDocument Create(System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap, Action<BsonDocumentBuilder> buildAction)
|
public static BsonDocument Create(ConcurrentDictionary<string, ushort> keyMap,
|
||||||
|
Action<BsonDocumentBuilder> buildAction)
|
||||||
{
|
{
|
||||||
var builder = new BsonDocumentBuilder(keyMap);
|
var builder = new BsonDocumentBuilder(keyMap);
|
||||||
buildAction(builder);
|
buildAction(builder);
|
||||||
return builder.Build();
|
return builder.Build();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Builder for creating BSON documents
|
/// Builder for creating BSON documents
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class BsonDocumentBuilder
|
public sealed class BsonDocumentBuilder
|
||||||
{
|
{
|
||||||
private byte[] _buffer = new byte[1024]; // Start with 1KB
|
private readonly ConcurrentDictionary<string, ushort> _keyMap;
|
||||||
private int _position;
|
private byte[] _buffer = new byte[1024]; // Start with 1KB
|
||||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, ushort> _keyMap;
|
private int _position;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BsonDocumentBuilder"/> class.
|
/// Initializes a new instance of the <see cref="BsonDocumentBuilder" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keyMap">The key map used for field name encoding.</param>
|
/// <param name="keyMap">The key map used for field name encoding.</param>
|
||||||
public BsonDocumentBuilder(System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap)
|
public BsonDocumentBuilder(ConcurrentDictionary<string, ushort> keyMap)
|
||||||
{
|
{
|
||||||
_keyMap = keyMap;
|
_keyMap = keyMap;
|
||||||
var writer = new BsonSpanWriter(_buffer, _keyMap);
|
var writer = new BsonSpanWriter(_buffer, _keyMap);
|
||||||
@@ -189,7 +194,7 @@ public sealed class BsonDocumentBuilder
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a string field to the document.
|
/// Adds a string field to the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The field value.</param>
|
/// <param name="value">The field value.</param>
|
||||||
@@ -204,7 +209,7 @@ public sealed class BsonDocumentBuilder
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds an Int32 field to the document.
|
/// Adds an Int32 field to the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The field value.</param>
|
/// <param name="value">The field value.</param>
|
||||||
@@ -213,13 +218,13 @@ public sealed class BsonDocumentBuilder
|
|||||||
{
|
{
|
||||||
EnsureCapacity(64);
|
EnsureCapacity(64);
|
||||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||||
writer.WriteInt32(name, value);
|
writer.WriteInt32(name, value);
|
||||||
_position += writer.Position;
|
_position += writer.Position;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds an Int64 field to the document.
|
/// Adds an Int64 field to the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The field value.</param>
|
/// <param name="value">The field value.</param>
|
||||||
@@ -228,13 +233,13 @@ public sealed class BsonDocumentBuilder
|
|||||||
{
|
{
|
||||||
EnsureCapacity(64);
|
EnsureCapacity(64);
|
||||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||||
writer.WriteInt64(name, value);
|
writer.WriteInt64(name, value);
|
||||||
_position += writer.Position;
|
_position += writer.Position;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a Boolean field to the document.
|
/// Adds a Boolean field to the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The field value.</param>
|
/// <param name="value">The field value.</param>
|
||||||
@@ -243,13 +248,13 @@ public sealed class BsonDocumentBuilder
|
|||||||
{
|
{
|
||||||
EnsureCapacity(64);
|
EnsureCapacity(64);
|
||||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||||
writer.WriteBoolean(name, value);
|
writer.WriteBoolean(name, value);
|
||||||
_position += writer.Position;
|
_position += writer.Position;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds an ObjectId field to the document.
|
/// Adds an ObjectId field to the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The field value.</param>
|
/// <param name="value">The field value.</param>
|
||||||
@@ -258,19 +263,19 @@ public sealed class BsonDocumentBuilder
|
|||||||
{
|
{
|
||||||
EnsureCapacity(64);
|
EnsureCapacity(64);
|
||||||
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
var writer = new BsonSpanWriter(_buffer.AsSpan(_position..), _keyMap);
|
||||||
writer.WriteObjectId(name, value);
|
writer.WriteObjectId(name, value);
|
||||||
_position += writer.Position;
|
_position += writer.Position;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Builds a BSON document from the accumulated fields.
|
/// Builds a BSON document from the accumulated fields.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The constructed BSON document.</returns>
|
/// <returns>The constructed BSON document.</returns>
|
||||||
public BsonDocument Build()
|
public BsonDocument Build()
|
||||||
{
|
{
|
||||||
// Layout: [int32 size][field bytes...][0x00 terminator]
|
// Layout: [int32 size][field bytes...][0x00 terminator]
|
||||||
var totalSize = _position + 5;
|
int totalSize = _position + 5;
|
||||||
var finalBuffer = new byte[totalSize];
|
var finalBuffer = new byte[totalSize];
|
||||||
|
|
||||||
BitConverter.TryWriteBytes(finalBuffer.AsSpan(0, 4), totalSize);
|
BitConverter.TryWriteBytes(finalBuffer.AsSpan(0, 4), totalSize);
|
||||||
@@ -279,14 +284,14 @@ public sealed class BsonDocumentBuilder
|
|||||||
|
|
||||||
return new BsonDocument(finalBuffer);
|
return new BsonDocument(finalBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void EnsureCapacity(int additional)
|
private void EnsureCapacity(int additional)
|
||||||
{
|
{
|
||||||
if (_position + additional > _buffer.Length)
|
if (_position + additional > _buffer.Length)
|
||||||
{
|
{
|
||||||
var newBuffer = new byte[_buffer.Length * 2];
|
var newBuffer = new byte[_buffer.Length * 2];
|
||||||
_buffer.CopyTo(newBuffer, 0);
|
_buffer.CopyTo(newBuffer, 0);
|
||||||
_buffer = newBuffer;
|
_buffer = newBuffer;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// BSON type codes as defined in BSON spec
|
/// BSON type codes as defined in BSON spec
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum BsonType : byte
|
public enum BsonType : byte
|
||||||
{
|
{
|
||||||
@@ -27,4 +27,4 @@ public enum BsonType : byte
|
|||||||
Decimal128 = 0x13,
|
Decimal128 = 0x13,
|
||||||
MinKey = 0xFF,
|
MinKey = 0xFF,
|
||||||
MaxKey = 0x7F
|
MaxKey = 0x7F
|
||||||
}
|
}
|
||||||
@@ -1,262 +1,263 @@
|
|||||||
using System;
|
using System.Buffers;
|
||||||
using System.Buffers;
|
using System.Buffers.Binary;
|
||||||
using System.Buffers.Binary;
|
using System.Text;
|
||||||
using System.Text;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// BSON writer that serializes to an IBufferWriter, enabling streaming serialization
|
||||||
/// BSON writer that serializes to an IBufferWriter, enabling streaming serialization
|
/// without fixed buffer size limits.
|
||||||
/// without fixed buffer size limits.
|
/// </summary>
|
||||||
/// </summary>
|
|
||||||
public ref struct BsonBufferWriter
|
public ref struct BsonBufferWriter
|
||||||
{
|
{
|
||||||
private IBufferWriter<byte> _writer;
|
private readonly IBufferWriter<byte> _writer;
|
||||||
private int _totalBytesWritten;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BsonBufferWriter"/> struct.
|
/// Initializes a new instance of the <see cref="BsonBufferWriter" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="writer">The buffer writer to write BSON bytes to.</param>
|
/// <param name="writer">The buffer writer to write BSON bytes to.</param>
|
||||||
public BsonBufferWriter(IBufferWriter<byte> writer)
|
public BsonBufferWriter(IBufferWriter<byte> writer)
|
||||||
{
|
{
|
||||||
_writer = writer;
|
_writer = writer;
|
||||||
_totalBytesWritten = 0;
|
Position = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current write position in bytes.
|
/// Gets the current write position in bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Position => _totalBytesWritten;
|
public int Position { get; private set; }
|
||||||
|
|
||||||
private void WriteBytes(ReadOnlySpan<byte> data)
|
private void WriteBytes(ReadOnlySpan<byte> data)
|
||||||
{
|
{
|
||||||
var destination = _writer.GetSpan(data.Length);
|
var destination = _writer.GetSpan(data.Length);
|
||||||
data.CopyTo(destination);
|
data.CopyTo(destination);
|
||||||
_writer.Advance(data.Length);
|
_writer.Advance(data.Length);
|
||||||
_totalBytesWritten += data.Length;
|
Position += data.Length;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WriteByte(byte value)
|
private void WriteByte(byte value)
|
||||||
{
|
{
|
||||||
var span = _writer.GetSpan(1);
|
var span = _writer.GetSpan(1);
|
||||||
span[0] = value;
|
span[0] = value;
|
||||||
_writer.Advance(1);
|
_writer.Advance(1);
|
||||||
_totalBytesWritten++;
|
Position++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON date-time field.
|
/// Writes a BSON date-time field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The date-time value.</param>
|
/// <param name="value">The date-time value.</param>
|
||||||
public void WriteDateTime(string name, DateTime value)
|
public void WriteDateTime(string name, DateTime value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.DateTime);
|
WriteByte((byte)BsonType.DateTime);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
// BSON DateTime: milliseconds since Unix epoch (UTC)
|
// BSON DateTime: milliseconds since Unix epoch (UTC)
|
||||||
var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||||
var milliseconds = (long)(value.ToUniversalTime() - unixEpoch).TotalMilliseconds;
|
var milliseconds = (long)(value.ToUniversalTime() - unixEpoch).TotalMilliseconds;
|
||||||
WriteInt64Internal(milliseconds);
|
WriteInt64Internal(milliseconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing a BSON document.
|
/// Begins writing a BSON document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The position where the document size placeholder was written.</returns>
|
/// <returns>The position where the document size placeholder was written.</returns>
|
||||||
public int BeginDocument()
|
public int BeginDocument()
|
||||||
{
|
{
|
||||||
// Write placeholder for size (4 bytes)
|
// Write placeholder for size (4 bytes)
|
||||||
var sizePosition = _totalBytesWritten;
|
int sizePosition = Position;
|
||||||
var span = _writer.GetSpan(4);
|
var span = _writer.GetSpan(4);
|
||||||
// Initialize with default value (will be patched later)
|
// Initialize with default value (will be patched later)
|
||||||
span[0] = 0; span[1] = 0; span[2] = 0; span[3] = 0;
|
span[0] = 0;
|
||||||
_writer.Advance(4);
|
span[1] = 0;
|
||||||
_totalBytesWritten += 4;
|
span[2] = 0;
|
||||||
return sizePosition;
|
span[3] = 0;
|
||||||
|
_writer.Advance(4);
|
||||||
|
Position += 4;
|
||||||
|
return sizePosition;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ends the current BSON document by writing the document terminator.
|
/// Ends the current BSON document by writing the document terminator.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="sizePosition">The position of the size placeholder for this document.</param>
|
/// <param name="sizePosition">The position of the size placeholder for this document.</param>
|
||||||
public void EndDocument(int sizePosition)
|
public void EndDocument(int sizePosition)
|
||||||
{
|
{
|
||||||
// Write document terminator
|
// Write document terminator
|
||||||
WriteByte(0);
|
WriteByte(0);
|
||||||
|
|
||||||
// Note: Size patching must be done by caller after accessing WrittenSpan
|
// Note: Size patching must be done by caller after accessing WrittenSpan
|
||||||
// from ArrayBufferWriter (or equivalent)
|
// from ArrayBufferWriter (or equivalent)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing a nested BSON document field.
|
/// Begins writing a nested BSON document field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <returns>The position where the nested document size placeholder was written.</returns>
|
/// <returns>The position where the nested document size placeholder was written.</returns>
|
||||||
public int BeginDocument(string name)
|
public int BeginDocument(string name)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Document);
|
WriteByte((byte)BsonType.Document);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
return BeginDocument();
|
return BeginDocument();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing a BSON array field.
|
/// Begins writing a BSON array field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <returns>The position where the array document size placeholder was written.</returns>
|
/// <returns>The position where the array document size placeholder was written.</returns>
|
||||||
public int BeginArray(string name)
|
public int BeginArray(string name)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Array);
|
WriteByte((byte)BsonType.Array);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
return BeginDocument();
|
return BeginDocument();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ends the current BSON array.
|
/// Ends the current BSON array.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="sizePosition">The position of the size placeholder for this array.</param>
|
/// <param name="sizePosition">The position of the size placeholder for this array.</param>
|
||||||
public void EndArray(int sizePosition)
|
public void EndArray(int sizePosition)
|
||||||
{
|
{
|
||||||
EndDocument(sizePosition);
|
EndDocument(sizePosition);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Private helper methods
|
// Private helper methods
|
||||||
|
|
||||||
private void WriteInt32Internal(int value)
|
private void WriteInt32Internal(int value)
|
||||||
{
|
{
|
||||||
var span = _writer.GetSpan(4);
|
var span = _writer.GetSpan(4);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(span, value);
|
BinaryPrimitives.WriteInt32LittleEndian(span, value);
|
||||||
_writer.Advance(4);
|
_writer.Advance(4);
|
||||||
_totalBytesWritten += 4;
|
Position += 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WriteInt64Internal(long value)
|
private void WriteInt64Internal(long value)
|
||||||
{
|
{
|
||||||
var span = _writer.GetSpan(8);
|
var span = _writer.GetSpan(8);
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(span, value);
|
BinaryPrimitives.WriteInt64LittleEndian(span, value);
|
||||||
_writer.Advance(8);
|
_writer.Advance(8);
|
||||||
_totalBytesWritten += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON ObjectId field.
|
/// Writes a BSON ObjectId field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The ObjectId value.</param>
|
/// <param name="value">The ObjectId value.</param>
|
||||||
public void WriteObjectId(string name, ObjectId value)
|
public void WriteObjectId(string name, ObjectId value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.ObjectId);
|
WriteByte((byte)BsonType.ObjectId);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteBytes(value.ToByteArray());
|
WriteBytes(value.ToByteArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON string field.
|
/// Writes a BSON string field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The string value.</param>
|
/// <param name="value">The string value.</param>
|
||||||
public void WriteString(string name, string value)
|
public void WriteString(string name, string value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.String);
|
WriteByte((byte)BsonType.String);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteStringValue(value);
|
WriteStringValue(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON boolean field.
|
/// Writes a BSON boolean field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The boolean value.</param>
|
/// <param name="value">The boolean value.</param>
|
||||||
public void WriteBoolean(string name, bool value)
|
public void WriteBoolean(string name, bool value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Boolean);
|
WriteByte((byte)BsonType.Boolean);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteByte((byte)(value ? 1 : 0));
|
WriteByte((byte)(value ? 1 : 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON null field.
|
/// Writes a BSON null field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
public void WriteNull(string name)
|
public void WriteNull(string name)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Null);
|
WriteByte((byte)BsonType.Null);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
}
|
|
||||||
|
|
||||||
private void WriteStringValue(string value)
|
|
||||||
{
|
|
||||||
// String: length (int32) + UTF8 bytes + null terminator
|
|
||||||
var bytes = Encoding.UTF8.GetBytes(value);
|
|
||||||
WriteInt32Internal(bytes.Length + 1); // +1 for null terminator
|
|
||||||
WriteBytes(bytes);
|
|
||||||
WriteByte(0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WriteDoubleInternal(double value)
|
private void WriteStringValue(string value)
|
||||||
{
|
{
|
||||||
var span = _writer.GetSpan(8);
|
// String: length (int32) + UTF8 bytes + null terminator
|
||||||
BinaryPrimitives.WriteDoubleLittleEndian(span, value);
|
byte[] bytes = Encoding.UTF8.GetBytes(value);
|
||||||
_writer.Advance(8);
|
WriteInt32Internal(bytes.Length + 1); // +1 for null terminator
|
||||||
_totalBytesWritten += 8;
|
WriteBytes(bytes);
|
||||||
|
WriteByte(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WriteDoubleInternal(double value)
|
||||||
|
{
|
||||||
|
var span = _writer.GetSpan(8);
|
||||||
|
BinaryPrimitives.WriteDoubleLittleEndian(span, value);
|
||||||
|
_writer.Advance(8);
|
||||||
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON binary field.
|
/// Writes a BSON binary field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="data">The binary data.</param>
|
/// <param name="data">The binary data.</param>
|
||||||
public void WriteBinary(string name, ReadOnlySpan<byte> data)
|
public void WriteBinary(string name, ReadOnlySpan<byte> data)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Binary);
|
WriteByte((byte)BsonType.Binary);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteInt32Internal(data.Length);
|
WriteInt32Internal(data.Length);
|
||||||
WriteByte(0); // Binary subtype: Generic
|
WriteByte(0); // Binary subtype: Generic
|
||||||
WriteBytes(data);
|
WriteBytes(data);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON 64-bit integer field.
|
/// Writes a BSON 64-bit integer field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The 64-bit integer value.</param>
|
/// <param name="value">The 64-bit integer value.</param>
|
||||||
public void WriteInt64(string name, long value)
|
public void WriteInt64(string name, long value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Int64);
|
WriteByte((byte)BsonType.Int64);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteInt64Internal(value);
|
WriteInt64Internal(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON double field.
|
/// Writes a BSON double field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The double value.</param>
|
/// <param name="value">The double value.</param>
|
||||||
public void WriteDouble(string name, double value)
|
public void WriteDouble(string name, double value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Double);
|
WriteByte((byte)BsonType.Double);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteDoubleInternal(value);
|
WriteDoubleInternal(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WriteCString(string value)
|
||||||
|
{
|
||||||
|
byte[] bytes = Encoding.UTF8.GetBytes(value);
|
||||||
|
WriteBytes(bytes);
|
||||||
|
WriteByte(0); // Null terminator
|
||||||
}
|
}
|
||||||
|
|
||||||
private void WriteCString(string value)
|
|
||||||
{
|
|
||||||
var bytes = Encoding.UTF8.GetBytes(value);
|
|
||||||
WriteBytes(bytes);
|
|
||||||
WriteByte(0); // Null terminator
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON 32-bit integer field.
|
/// Writes a BSON 32-bit integer field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The 32-bit integer value.</param>
|
/// <param name="value">The 32-bit integer value.</param>
|
||||||
public void WriteInt32(string name, int value)
|
public void WriteInt32(string name, int value)
|
||||||
{
|
{
|
||||||
WriteByte((byte)BsonType.Int32);
|
WriteByte((byte)BsonType.Int32);
|
||||||
WriteCString(name);
|
WriteCString(name);
|
||||||
WriteInt32Internal(value);
|
WriteInt32Internal(value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,344 +1,343 @@
|
|||||||
using System;
|
|
||||||
using System.Buffers.Binary;
|
using System.Buffers.Binary;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Zero-allocation BSON reader using ReadOnlySpan<byte>.
|
/// Zero-allocation BSON reader using ReadOnlySpan<byte>.
|
||||||
/// Implemented as ref struct to ensure stack-only allocation.
|
/// Implemented as ref struct to ensure stack-only allocation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ref struct BsonSpanReader
|
public ref struct BsonSpanReader
|
||||||
{
|
{
|
||||||
private ReadOnlySpan<byte> _buffer;
|
private ReadOnlySpan<byte> _buffer;
|
||||||
private int _position;
|
private readonly ConcurrentDictionary<ushort, string> _keys;
|
||||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<ushort, string> _keys;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new instance of the <see cref="BsonSpanReader"/> struct.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="buffer">The BSON buffer to read.</param>
|
|
||||||
/// <param name="keys">The reverse key dictionary used for compressed element headers.</param>
|
|
||||||
public BsonSpanReader(ReadOnlySpan<byte> buffer, System.Collections.Concurrent.ConcurrentDictionary<ushort, string> keys)
|
|
||||||
{
|
|
||||||
_buffer = buffer;
|
|
||||||
_position = 0;
|
|
||||||
_keys = keys;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the current read position in the buffer.
|
|
||||||
/// </summary>
|
|
||||||
public int Position => _position;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the number of unread bytes remaining in the buffer.
|
|
||||||
/// </summary>
|
|
||||||
public int Remaining => _buffer.Length - _position;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads the document size (first 4 bytes of a BSON document)
|
/// Initializes a new instance of the <see cref="BsonSpanReader" /> struct.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="buffer">The BSON buffer to read.</param>
|
||||||
|
/// <param name="keys">The reverse key dictionary used for compressed element headers.</param>
|
||||||
|
public BsonSpanReader(ReadOnlySpan<byte> buffer, ConcurrentDictionary<ushort, string> keys)
|
||||||
|
{
|
||||||
|
_buffer = buffer;
|
||||||
|
Position = 0;
|
||||||
|
_keys = keys;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the current read position in the buffer.
|
||||||
|
/// </summary>
|
||||||
|
public int Position { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the number of unread bytes remaining in the buffer.
|
||||||
|
/// </summary>
|
||||||
|
public int Remaining => _buffer.Length - Position;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads the document size (first 4 bytes of a BSON document)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int ReadDocumentSize()
|
public int ReadDocumentSize()
|
||||||
{
|
{
|
||||||
if (Remaining < 4)
|
if (Remaining < 4)
|
||||||
throw new InvalidOperationException("Not enough bytes to read document size");
|
throw new InvalidOperationException("Not enough bytes to read document size");
|
||||||
|
|
||||||
var size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
int size = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
_position += 4;
|
Position += 4;
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a BSON element type
|
/// Reads a BSON element type
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public BsonType ReadBsonType()
|
public BsonType ReadBsonType()
|
||||||
{
|
{
|
||||||
if (Remaining < 1)
|
if (Remaining < 1)
|
||||||
throw new InvalidOperationException("Not enough bytes to read BSON type");
|
throw new InvalidOperationException("Not enough bytes to read BSON type");
|
||||||
|
|
||||||
var type = (BsonType)_buffer[_position];
|
var type = (BsonType)_buffer[Position];
|
||||||
_position++;
|
Position++;
|
||||||
return type;
|
return type;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a C-style null-terminated string (e-name in BSON spec)
|
/// Reads a C-style null-terminated string (e-name in BSON spec)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string ReadCString()
|
public string ReadCString()
|
||||||
{
|
{
|
||||||
var start = _position;
|
int start = Position;
|
||||||
while (_position < _buffer.Length && _buffer[_position] != 0)
|
while (Position < _buffer.Length && _buffer[Position] != 0)
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
if (_position >= _buffer.Length)
|
if (Position >= _buffer.Length)
|
||||||
throw new InvalidOperationException("Unterminated C-string");
|
throw new InvalidOperationException("Unterminated C-string");
|
||||||
|
|
||||||
var nameBytes = _buffer.Slice(start, _position - start);
|
var nameBytes = _buffer.Slice(start, Position - start);
|
||||||
_position++; // Skip null terminator
|
Position++; // Skip null terminator
|
||||||
|
|
||||||
return Encoding.UTF8.GetString(nameBytes);
|
return Encoding.UTF8.GetString(nameBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a C-string into a destination span. Returns the number of bytes written.
|
/// Reads a C-string into a destination span. Returns the number of bytes written.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination character span.</param>
|
/// <param name="destination">The destination character span.</param>
|
||||||
public int ReadCString(Span<char> destination)
|
public int ReadCString(Span<char> destination)
|
||||||
{
|
{
|
||||||
var start = _position;
|
int start = Position;
|
||||||
while (_position < _buffer.Length && _buffer[_position] != 0)
|
while (Position < _buffer.Length && _buffer[Position] != 0)
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
if (_position >= _buffer.Length)
|
if (Position >= _buffer.Length)
|
||||||
throw new InvalidOperationException("Unterminated C-string");
|
throw new InvalidOperationException("Unterminated C-string");
|
||||||
|
|
||||||
var nameBytes = _buffer.Slice(start, _position - start);
|
var nameBytes = _buffer.Slice(start, Position - start);
|
||||||
_position++; // Skip null terminator
|
Position++; // Skip null terminator
|
||||||
|
|
||||||
return Encoding.UTF8.GetChars(nameBytes, destination);
|
return Encoding.UTF8.GetChars(nameBytes, destination);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a BSON string (4-byte length + UTF-8 bytes + null terminator)
|
/// Reads a BSON string (4-byte length + UTF-8 bytes + null terminator)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string ReadString()
|
public string ReadString()
|
||||||
{
|
{
|
||||||
var length = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
int length = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
_position += 4;
|
Position += 4;
|
||||||
|
|
||||||
if (length < 1)
|
if (length < 1)
|
||||||
throw new InvalidOperationException("Invalid string length");
|
throw new InvalidOperationException("Invalid string length");
|
||||||
|
|
||||||
var stringBytes = _buffer.Slice(_position, length - 1); // Exclude null terminator
|
var stringBytes = _buffer.Slice(Position, length - 1); // Exclude null terminator
|
||||||
_position += length;
|
Position += length;
|
||||||
|
|
||||||
return Encoding.UTF8.GetString(stringBytes);
|
return Encoding.UTF8.GetString(stringBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a 32-bit integer.
|
/// Reads a 32-bit integer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int ReadInt32()
|
public int ReadInt32()
|
||||||
{
|
{
|
||||||
if (Remaining < 4)
|
if (Remaining < 4)
|
||||||
throw new InvalidOperationException("Not enough bytes to read Int32");
|
throw new InvalidOperationException("Not enough bytes to read Int32");
|
||||||
|
|
||||||
var value = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
int value = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
_position += 4;
|
Position += 4;
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Reads a 64-bit integer.
|
|
||||||
/// </summary>
|
|
||||||
public long ReadInt64()
|
|
||||||
{
|
|
||||||
if (Remaining < 8)
|
|
||||||
throw new InvalidOperationException("Not enough bytes to read Int64");
|
|
||||||
|
|
||||||
var value = BinaryPrimitives.ReadInt64LittleEndian(_buffer.Slice(_position, 8));
|
|
||||||
_position += 8;
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Reads a double-precision floating point value.
|
|
||||||
/// </summary>
|
|
||||||
public double ReadDouble()
|
|
||||||
{
|
|
||||||
if (Remaining < 8)
|
|
||||||
throw new InvalidOperationException("Not enough bytes to read Double");
|
|
||||||
|
|
||||||
var value = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
|
||||||
_position += 8;
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads spatial coordinates from a BSON array [X, Y].
|
/// Reads a 64-bit integer.
|
||||||
/// Returns a (double, double) tuple.
|
/// </summary>
|
||||||
|
public long ReadInt64()
|
||||||
|
{
|
||||||
|
if (Remaining < 8)
|
||||||
|
throw new InvalidOperationException("Not enough bytes to read Int64");
|
||||||
|
|
||||||
|
long value = BinaryPrimitives.ReadInt64LittleEndian(_buffer.Slice(Position, 8));
|
||||||
|
Position += 8;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads a double-precision floating point value.
|
||||||
|
/// </summary>
|
||||||
|
public double ReadDouble()
|
||||||
|
{
|
||||||
|
if (Remaining < 8)
|
||||||
|
throw new InvalidOperationException("Not enough bytes to read Double");
|
||||||
|
|
||||||
|
double value = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(Position, 8));
|
||||||
|
Position += 8;
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Reads spatial coordinates from a BSON array [X, Y].
|
||||||
|
/// Returns a (double, double) tuple.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public (double, double) ReadCoordinates()
|
public (double, double) ReadCoordinates()
|
||||||
{
|
{
|
||||||
// Skip array size (4 bytes)
|
// Skip array size (4 bytes)
|
||||||
_position += 4;
|
Position += 4;
|
||||||
|
|
||||||
// Skip element 0 header: Type(1) + Name("0\0") (3 bytes)
|
// Skip element 0 header: Type(1) + Name("0\0") (3 bytes)
|
||||||
_position += 3;
|
Position += 3;
|
||||||
var x = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
double x = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(Position, 8));
|
||||||
_position += 8;
|
Position += 8;
|
||||||
|
|
||||||
// Skip element 1 header: Type(1) + Name("1\0") (3 bytes)
|
// Skip element 1 header: Type(1) + Name("1\0") (3 bytes)
|
||||||
_position += 3;
|
Position += 3;
|
||||||
var y = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(_position, 8));
|
double y = BinaryPrimitives.ReadDoubleLittleEndian(_buffer.Slice(Position, 8));
|
||||||
_position += 8;
|
Position += 8;
|
||||||
|
|
||||||
// Skip end of array marker (1 byte)
|
// Skip end of array marker (1 byte)
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
return (x, y);
|
return (x, y);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a Decimal128 value.
|
/// Reads a Decimal128 value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public decimal ReadDecimal128()
|
public decimal ReadDecimal128()
|
||||||
{
|
{
|
||||||
if (Remaining < 16)
|
if (Remaining < 16)
|
||||||
throw new InvalidOperationException("Not enough bytes to read Decimal128");
|
throw new InvalidOperationException("Not enough bytes to read Decimal128");
|
||||||
|
|
||||||
var bits = new int[4];
|
var bits = new int[4];
|
||||||
bits[0] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
bits[0] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
bits[1] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 4, 4));
|
bits[1] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position + 4, 4));
|
||||||
bits[2] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 8, 4));
|
bits[2] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position + 8, 4));
|
||||||
bits[3] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position + 12, 4));
|
bits[3] = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position + 12, 4));
|
||||||
_position += 16;
|
Position += 16;
|
||||||
|
|
||||||
return new decimal(bits);
|
return new decimal(bits);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a boolean value.
|
/// Reads a boolean value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool ReadBoolean()
|
public bool ReadBoolean()
|
||||||
{
|
{
|
||||||
if (Remaining < 1)
|
if (Remaining < 1)
|
||||||
throw new InvalidOperationException("Not enough bytes to read Boolean");
|
throw new InvalidOperationException("Not enough bytes to read Boolean");
|
||||||
|
|
||||||
var value = _buffer[_position] != 0;
|
bool value = _buffer[Position] != 0;
|
||||||
_position++;
|
Position++;
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a BSON DateTime (UTC milliseconds since Unix epoch)
|
/// Reads a BSON DateTime (UTC milliseconds since Unix epoch)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateTime ReadDateTime()
|
public DateTime ReadDateTime()
|
||||||
{
|
{
|
||||||
var milliseconds = ReadInt64();
|
long milliseconds = ReadInt64();
|
||||||
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds).UtcDateTime;
|
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds).UtcDateTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a BSON DateTime as DateTimeOffset (UTC milliseconds since Unix epoch)
|
/// Reads a BSON DateTime as DateTimeOffset (UTC milliseconds since Unix epoch)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateTimeOffset ReadDateTimeOffset()
|
public DateTimeOffset ReadDateTimeOffset()
|
||||||
{
|
{
|
||||||
var milliseconds = ReadInt64();
|
long milliseconds = ReadInt64();
|
||||||
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
|
return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a TimeSpan from BSON Int64 (ticks)
|
/// Reads a TimeSpan from BSON Int64 (ticks)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public TimeSpan ReadTimeSpan()
|
public TimeSpan ReadTimeSpan()
|
||||||
{
|
{
|
||||||
var ticks = ReadInt64();
|
long ticks = ReadInt64();
|
||||||
return TimeSpan.FromTicks(ticks);
|
return TimeSpan.FromTicks(ticks);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a DateOnly from BSON Int32 (day number)
|
/// Reads a DateOnly from BSON Int32 (day number)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateOnly ReadDateOnly()
|
public DateOnly ReadDateOnly()
|
||||||
{
|
{
|
||||||
var dayNumber = ReadInt32();
|
int dayNumber = ReadInt32();
|
||||||
return DateOnly.FromDayNumber(dayNumber);
|
return DateOnly.FromDayNumber(dayNumber);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a TimeOnly from BSON Int64 (ticks)
|
/// Reads a TimeOnly from BSON Int64 (ticks)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public TimeOnly ReadTimeOnly()
|
public TimeOnly ReadTimeOnly()
|
||||||
{
|
{
|
||||||
var ticks = ReadInt64();
|
long ticks = ReadInt64();
|
||||||
return new TimeOnly(ticks);
|
return new TimeOnly(ticks);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a GUID value.
|
/// Reads a GUID value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Guid ReadGuid()
|
public Guid ReadGuid()
|
||||||
{
|
{
|
||||||
return Guid.Parse(ReadString());
|
return Guid.Parse(ReadString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a BSON ObjectId (12 bytes)
|
/// Reads a BSON ObjectId (12 bytes)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ObjectId ReadObjectId()
|
public ObjectId ReadObjectId()
|
||||||
{
|
{
|
||||||
if (Remaining < 12)
|
if (Remaining < 12)
|
||||||
throw new InvalidOperationException("Not enough bytes to read ObjectId");
|
throw new InvalidOperationException("Not enough bytes to read ObjectId");
|
||||||
|
|
||||||
var oidBytes = _buffer.Slice(_position, 12);
|
var oidBytes = _buffer.Slice(Position, 12);
|
||||||
_position += 12;
|
Position += 12;
|
||||||
return new ObjectId(oidBytes);
|
return new ObjectId(oidBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads binary data (subtype + length + bytes)
|
/// Reads binary data (subtype + length + bytes)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="subtype">When this method returns, contains the BSON binary subtype.</param>
|
/// <param name="subtype">When this method returns, contains the BSON binary subtype.</param>
|
||||||
public ReadOnlySpan<byte> ReadBinary(out byte subtype)
|
public ReadOnlySpan<byte> ReadBinary(out byte subtype)
|
||||||
{
|
{
|
||||||
var length = ReadInt32();
|
int length = ReadInt32();
|
||||||
|
|
||||||
if (Remaining < 1)
|
if (Remaining < 1)
|
||||||
throw new InvalidOperationException("Not enough bytes to read binary subtype");
|
throw new InvalidOperationException("Not enough bytes to read binary subtype");
|
||||||
|
|
||||||
subtype = _buffer[_position];
|
subtype = _buffer[Position];
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
if (Remaining < length)
|
if (Remaining < length)
|
||||||
throw new InvalidOperationException("Not enough bytes to read binary data");
|
throw new InvalidOperationException("Not enough bytes to read binary data");
|
||||||
|
|
||||||
var data = _buffer.Slice(_position, length);
|
var data = _buffer.Slice(Position, length);
|
||||||
_position += length;
|
Position += length;
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Skips the current value based on type
|
/// Skips the current value based on type
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="type">The BSON type of the value to skip.</param>
|
/// <param name="type">The BSON type of the value to skip.</param>
|
||||||
public void SkipValue(BsonType type)
|
public void SkipValue(BsonType type)
|
||||||
{
|
{
|
||||||
switch (type)
|
switch (type)
|
||||||
{
|
{
|
||||||
case BsonType.Double:
|
case BsonType.Double:
|
||||||
_position += 8;
|
Position += 8;
|
||||||
break;
|
break;
|
||||||
case BsonType.String:
|
case BsonType.String:
|
||||||
var stringLength = ReadInt32();
|
int stringLength = ReadInt32();
|
||||||
_position += stringLength;
|
Position += stringLength;
|
||||||
break;
|
break;
|
||||||
case BsonType.Document:
|
case BsonType.Document:
|
||||||
case BsonType.Array:
|
case BsonType.Array:
|
||||||
var docLength = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
int docLength = BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
_position += docLength;
|
Position += docLength;
|
||||||
break;
|
break;
|
||||||
case BsonType.Binary:
|
case BsonType.Binary:
|
||||||
var binaryLength = ReadInt32();
|
int binaryLength = ReadInt32();
|
||||||
_position += 1 + binaryLength; // subtype + data
|
Position += 1 + binaryLength; // subtype + data
|
||||||
break;
|
break;
|
||||||
case BsonType.ObjectId:
|
case BsonType.ObjectId:
|
||||||
_position += 12;
|
Position += 12;
|
||||||
break;
|
break;
|
||||||
case BsonType.Boolean:
|
case BsonType.Boolean:
|
||||||
_position += 1;
|
Position += 1;
|
||||||
break;
|
break;
|
||||||
case BsonType.DateTime:
|
case BsonType.DateTime:
|
||||||
case BsonType.Int64:
|
case BsonType.Int64:
|
||||||
case BsonType.Timestamp:
|
case BsonType.Timestamp:
|
||||||
_position += 8;
|
Position += 8;
|
||||||
break;
|
break;
|
||||||
case BsonType.Decimal128:
|
case BsonType.Decimal128:
|
||||||
_position += 16;
|
Position += 16;
|
||||||
break;
|
break;
|
||||||
case BsonType.Int32:
|
case BsonType.Int32:
|
||||||
_position += 4;
|
Position += 4;
|
||||||
break;
|
break;
|
||||||
case BsonType.Null:
|
case BsonType.Null:
|
||||||
// No data
|
// No data
|
||||||
@@ -348,49 +347,50 @@ public ref struct BsonSpanReader
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a single byte.
|
/// Reads a single byte.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public byte ReadByte()
|
public byte ReadByte()
|
||||||
{
|
{
|
||||||
if (Remaining < 1)
|
if (Remaining < 1)
|
||||||
throw new InvalidOperationException("Not enough bytes to read byte");
|
throw new InvalidOperationException("Not enough bytes to read byte");
|
||||||
var value = _buffer[_position];
|
byte value = _buffer[Position];
|
||||||
_position++;
|
Position++;
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Peeks a 32-bit integer at the current position without advancing.
|
/// Peeks a 32-bit integer at the current position without advancing.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int PeekInt32()
|
public int PeekInt32()
|
||||||
{
|
{
|
||||||
if (Remaining < 4)
|
if (Remaining < 4)
|
||||||
throw new InvalidOperationException("Not enough bytes to peek Int32");
|
throw new InvalidOperationException("Not enough bytes to peek Int32");
|
||||||
return BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(_position, 4));
|
return BinaryPrimitives.ReadInt32LittleEndian(_buffer.Slice(Position, 4));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads an element header key identifier and resolves it to a key name.
|
/// Reads an element header key identifier and resolves it to a key name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string ReadElementHeader()
|
public string ReadElementHeader()
|
||||||
{
|
{
|
||||||
if (Remaining < 2)
|
if (Remaining < 2)
|
||||||
throw new InvalidOperationException("Not enough bytes to read BSON element key ID");
|
throw new InvalidOperationException("Not enough bytes to read BSON element key ID");
|
||||||
|
|
||||||
var id = BinaryPrimitives.ReadUInt16LittleEndian(_buffer.Slice(_position, 2));
|
ushort id = BinaryPrimitives.ReadUInt16LittleEndian(_buffer.Slice(Position, 2));
|
||||||
_position += 2;
|
Position += 2;
|
||||||
|
|
||||||
if (!_keys.TryGetValue(id, out var key))
|
if (!_keys.TryGetValue(id, out string? key))
|
||||||
{
|
|
||||||
throw new InvalidOperationException($"BSON Key ID {id} not found in reverse key dictionary.");
|
throw new InvalidOperationException($"BSON Key ID {id} not found in reverse key dictionary.");
|
||||||
}
|
|
||||||
|
|
||||||
return key;
|
return key;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns a span containing all unread bytes.
|
/// Returns a span containing all unread bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ReadOnlySpan<byte> RemainingBytes() => _buffer[_position..];
|
public ReadOnlySpan<byte> RemainingBytes()
|
||||||
}
|
{
|
||||||
|
return _buffer[Position..];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,382 +1,380 @@
|
|||||||
using System;
|
|
||||||
using System.Buffers.Binary;
|
using System.Buffers.Binary;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Zero-allocation BSON writer using Span<byte>.
|
/// Zero-allocation BSON writer using Span<byte>.
|
||||||
/// Implemented as ref struct to ensure stack-only allocation.
|
/// Implemented as ref struct to ensure stack-only allocation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ref struct BsonSpanWriter
|
public ref struct BsonSpanWriter
|
||||||
{
|
{
|
||||||
private Span<byte> _buffer;
|
private Span<byte> _buffer;
|
||||||
private int _position;
|
private readonly ConcurrentDictionary<string, ushort> _keyMap;
|
||||||
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, ushort> _keyMap;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BsonSpanWriter"/> struct.
|
/// Initializes a new instance of the <see cref="BsonSpanWriter" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="buffer">The destination buffer to write BSON bytes into.</param>
|
/// <param name="buffer">The destination buffer to write BSON bytes into.</param>
|
||||||
/// <param name="keyMap">The cached key-name to key-id mapping.</param>
|
/// <param name="keyMap">The cached key-name to key-id mapping.</param>
|
||||||
public BsonSpanWriter(Span<byte> buffer, System.Collections.Concurrent.ConcurrentDictionary<string, ushort> keyMap)
|
public BsonSpanWriter(Span<byte> buffer, ConcurrentDictionary<string, ushort> keyMap)
|
||||||
{
|
{
|
||||||
_buffer = buffer;
|
_buffer = buffer;
|
||||||
_keyMap = keyMap;
|
_keyMap = keyMap;
|
||||||
_position = 0;
|
Position = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current write position in the buffer.
|
/// Gets the current write position in the buffer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Position => _position;
|
public int Position { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the number of bytes remaining in the buffer.
|
|
||||||
/// </summary>
|
|
||||||
public int Remaining => _buffer.Length - _position;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes document size placeholder and returns the position to patch later
|
/// Gets the number of bytes remaining in the buffer.
|
||||||
|
/// </summary>
|
||||||
|
public int Remaining => _buffer.Length - Position;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes document size placeholder and returns the position to patch later
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int WriteDocumentSizePlaceholder()
|
public int WriteDocumentSizePlaceholder()
|
||||||
{
|
{
|
||||||
var sizePosition = _position;
|
int sizePosition = Position;
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), 0);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), 0);
|
||||||
_position += 4;
|
Position += 4;
|
||||||
return sizePosition;
|
return sizePosition;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Patches the document size at the given position
|
/// Patches the document size at the given position
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="sizePosition">The position where the size placeholder was written.</param>
|
/// <param name="sizePosition">The position where the size placeholder was written.</param>
|
||||||
public void PatchDocumentSize(int sizePosition)
|
public void PatchDocumentSize(int sizePosition)
|
||||||
{
|
{
|
||||||
var size = _position - sizePosition;
|
int size = Position - sizePosition;
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(sizePosition, 4), size);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(sizePosition, 4), size);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON element header (type + name)
|
/// Writes a BSON element header (type + name)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="type">The BSON element type.</param>
|
/// <param name="type">The BSON element type.</param>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
public void WriteElementHeader(BsonType type, string name)
|
public void WriteElementHeader(BsonType type, string name)
|
||||||
{
|
{
|
||||||
_buffer[_position] = (byte)type;
|
_buffer[Position] = (byte)type;
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
if (!_keyMap.TryGetValue(name, out var id))
|
if (!_keyMap.TryGetValue(name, out ushort id))
|
||||||
{
|
throw new InvalidOperationException(
|
||||||
throw new InvalidOperationException($"BSON Key '{name}' not found in dictionary cache. Ensure all keys are registered before serialization.");
|
$"BSON Key '{name}' not found in dictionary cache. Ensure all keys are registered before serialization.");
|
||||||
}
|
|
||||||
|
|
||||||
BinaryPrimitives.WriteUInt16LittleEndian(_buffer.Slice(_position, 2), id);
|
BinaryPrimitives.WriteUInt16LittleEndian(_buffer.Slice(Position, 2), id);
|
||||||
_position += 2;
|
Position += 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a C-style null-terminated string
|
/// Writes a C-style null-terminated string
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private void WriteCString(string value)
|
private void WriteCString(string value)
|
||||||
{
|
{
|
||||||
var bytesWritten = Encoding.UTF8.GetBytes(value, _buffer[_position..]);
|
int bytesWritten = Encoding.UTF8.GetBytes(value, _buffer[Position..]);
|
||||||
_position += bytesWritten;
|
Position += bytesWritten;
|
||||||
_buffer[_position] = 0; // Null terminator
|
_buffer[Position] = 0; // Null terminator
|
||||||
_position++;
|
Position++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes end-of-document marker
|
/// Writes end-of-document marker
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void WriteEndOfDocument()
|
public void WriteEndOfDocument()
|
||||||
{
|
{
|
||||||
_buffer[_position] = 0;
|
_buffer[Position] = 0;
|
||||||
_position++;
|
Position++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON string element
|
/// Writes a BSON string element
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The string value.</param>
|
/// <param name="value">The string value.</param>
|
||||||
public void WriteString(string name, string value)
|
public void WriteString(string name, string value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.String, name);
|
WriteElementHeader(BsonType.String, name);
|
||||||
|
|
||||||
var valueBytes = Encoding.UTF8.GetByteCount(value);
|
int valueBytes = Encoding.UTF8.GetByteCount(value);
|
||||||
var stringLength = valueBytes + 1; // Include null terminator
|
int stringLength = valueBytes + 1; // Include null terminator
|
||||||
|
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), stringLength);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), stringLength);
|
||||||
_position += 4;
|
Position += 4;
|
||||||
|
|
||||||
Encoding.UTF8.GetBytes(value, _buffer[_position..]);
|
Encoding.UTF8.GetBytes(value, _buffer[Position..]);
|
||||||
_position += valueBytes;
|
Position += valueBytes;
|
||||||
|
|
||||||
_buffer[_position] = 0; // Null terminator
|
_buffer[Position] = 0; // Null terminator
|
||||||
_position++;
|
Position++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON int32 element.
|
/// Writes a BSON int32 element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The 32-bit integer value.</param>
|
/// <param name="value">The 32-bit integer value.</param>
|
||||||
public void WriteInt32(string name, int value)
|
public void WriteInt32(string name, int value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Int32, name);
|
WriteElementHeader(BsonType.Int32, name);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), value);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), value);
|
||||||
_position += 4;
|
Position += 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON int64 element.
|
/// Writes a BSON int64 element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The 64-bit integer value.</param>
|
/// <param name="value">The 64-bit integer value.</param>
|
||||||
public void WriteInt64(string name, long value)
|
public void WriteInt64(string name, long value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Int64, name);
|
WriteElementHeader(BsonType.Int64, name);
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value);
|
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(Position, 8), value);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON double element.
|
/// Writes a BSON double element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The double-precision value.</param>
|
/// <param name="value">The double-precision value.</param>
|
||||||
public void WriteDouble(string name, double value)
|
public void WriteDouble(string name, double value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Double, name);
|
WriteElementHeader(BsonType.Double, name);
|
||||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), value);
|
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(Position, 8), value);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes spatial coordinates as a BSON array [X, Y].
|
/// Writes spatial coordinates as a BSON array [X, Y].
|
||||||
/// Optimized for (double, double) tuples.
|
/// Optimized for (double, double) tuples.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="coordinates">The coordinate tuple as (X, Y).</param>
|
/// <param name="coordinates">The coordinate tuple as (X, Y).</param>
|
||||||
public void WriteCoordinates(string name, (double, double) coordinates)
|
public void WriteCoordinates(string name, (double, double) coordinates)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Array, name);
|
WriteElementHeader(BsonType.Array, name);
|
||||||
|
|
||||||
var startPos = _position;
|
int startPos = Position;
|
||||||
_position += 4; // Placeholder for array size
|
Position += 4; // Placeholder for array size
|
||||||
|
|
||||||
// Element 0: X
|
// Element 0: X
|
||||||
_buffer[_position++] = (byte)BsonType.Double;
|
_buffer[Position++] = (byte)BsonType.Double;
|
||||||
_buffer[_position++] = 0x30; // '0'
|
_buffer[Position++] = 0x30; // '0'
|
||||||
_buffer[_position++] = 0x00; // Null
|
_buffer[Position++] = 0x00; // Null
|
||||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), coordinates.Item1);
|
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(Position, 8), coordinates.Item1);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
|
|
||||||
// Element 1: Y
|
// Element 1: Y
|
||||||
_buffer[_position++] = (byte)BsonType.Double;
|
_buffer[Position++] = (byte)BsonType.Double;
|
||||||
_buffer[_position++] = 0x31; // '1'
|
_buffer[Position++] = 0x31; // '1'
|
||||||
_buffer[_position++] = 0x00; // Null
|
_buffer[Position++] = 0x00; // Null
|
||||||
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(_position, 8), coordinates.Item2);
|
BinaryPrimitives.WriteDoubleLittleEndian(_buffer.Slice(Position, 8), coordinates.Item2);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
|
|
||||||
_buffer[_position++] = 0x00; // End of array marker
|
_buffer[Position++] = 0x00; // End of array marker
|
||||||
|
|
||||||
// Patch array size
|
// Patch array size
|
||||||
var size = _position - startPos;
|
int size = Position - startPos;
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(startPos, 4), size);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(startPos, 4), size);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON Decimal128 element from a <see cref="decimal"/> value.
|
/// Writes a BSON Decimal128 element from a <see cref="decimal" /> value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The decimal value.</param>
|
/// <param name="value">The decimal value.</param>
|
||||||
public void WriteDecimal128(string name, decimal value)
|
public void WriteDecimal128(string name, decimal value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Decimal128, name);
|
WriteElementHeader(BsonType.Decimal128, name);
|
||||||
// Note: usage of C# decimal bits for round-trip fidelity within ZB.MOM.WW.CBDD.
|
// Note: usage of C# decimal bits for round-trip fidelity within ZB.MOM.WW.CBDD.
|
||||||
// This makes it compatible with CBDD Reader but strictly speaking not standard IEEE 754-2008 Decimal128.
|
// This makes it compatible with CBDD Reader but strictly speaking not standard IEEE 754-2008 Decimal128.
|
||||||
var bits = decimal.GetBits(value);
|
int[] bits = decimal.GetBits(value);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), bits[0]);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), bits[0]);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 4, 4), bits[1]);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position + 4, 4), bits[1]);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 8, 4), bits[2]);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position + 8, 4), bits[2]);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position + 12, 4), bits[3]);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position + 12, 4), bits[3]);
|
||||||
_position += 16;
|
Position += 16;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON boolean element.
|
/// Writes a BSON boolean element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The boolean value.</param>
|
/// <param name="value">The boolean value.</param>
|
||||||
public void WriteBoolean(string name, bool value)
|
public void WriteBoolean(string name, bool value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Boolean, name);
|
WriteElementHeader(BsonType.Boolean, name);
|
||||||
_buffer[_position] = (byte)(value ? 1 : 0);
|
_buffer[Position] = (byte)(value ? 1 : 0);
|
||||||
_position++;
|
Position++;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON UTC datetime element.
|
/// Writes a BSON UTC datetime element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The date and time value.</param>
|
/// <param name="value">The date and time value.</param>
|
||||||
public void WriteDateTime(string name, DateTime value)
|
public void WriteDateTime(string name, DateTime value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.DateTime, name);
|
WriteElementHeader(BsonType.DateTime, name);
|
||||||
var milliseconds = new DateTimeOffset(value.ToUniversalTime()).ToUnixTimeMilliseconds();
|
long milliseconds = new DateTimeOffset(value.ToUniversalTime()).ToUnixTimeMilliseconds();
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
|
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(Position, 8), milliseconds);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON UTC datetime element from a <see cref="DateTimeOffset"/> value.
|
/// Writes a BSON UTC datetime element from a <see cref="DateTimeOffset" /> value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The date and time offset value.</param>
|
/// <param name="value">The date and time offset value.</param>
|
||||||
public void WriteDateTimeOffset(string name, DateTimeOffset value)
|
public void WriteDateTimeOffset(string name, DateTimeOffset value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.DateTime, name);
|
WriteElementHeader(BsonType.DateTime, name);
|
||||||
var milliseconds = value.ToUnixTimeMilliseconds();
|
long milliseconds = value.ToUnixTimeMilliseconds();
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), milliseconds);
|
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(Position, 8), milliseconds);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON int64 element containing ticks from a <see cref="TimeSpan"/>.
|
/// Writes a BSON int64 element containing ticks from a <see cref="TimeSpan" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The time span value.</param>
|
/// <param name="value">The time span value.</param>
|
||||||
public void WriteTimeSpan(string name, TimeSpan value)
|
public void WriteTimeSpan(string name, TimeSpan value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Int64, name);
|
WriteElementHeader(BsonType.Int64, name);
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value.Ticks);
|
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(Position, 8), value.Ticks);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON int32 element containing the <see cref="DateOnly.DayNumber"/>.
|
/// Writes a BSON int32 element containing the <see cref="DateOnly.DayNumber" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The date-only value.</param>
|
/// <param name="value">The date-only value.</param>
|
||||||
public void WriteDateOnly(string name, DateOnly value)
|
public void WriteDateOnly(string name, DateOnly value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Int32, name);
|
WriteElementHeader(BsonType.Int32, name);
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), value.DayNumber);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), value.DayNumber);
|
||||||
_position += 4;
|
Position += 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON int64 element containing ticks from a <see cref="TimeOnly"/>.
|
/// Writes a BSON int64 element containing ticks from a <see cref="TimeOnly" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The time-only value.</param>
|
/// <param name="value">The time-only value.</param>
|
||||||
public void WriteTimeOnly(string name, TimeOnly value)
|
public void WriteTimeOnly(string name, TimeOnly value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Int64, name);
|
WriteElementHeader(BsonType.Int64, name);
|
||||||
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(_position, 8), value.Ticks);
|
BinaryPrimitives.WriteInt64LittleEndian(_buffer.Slice(Position, 8), value.Ticks);
|
||||||
_position += 8;
|
Position += 8;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a GUID as a BSON string element.
|
/// Writes a GUID as a BSON string element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The GUID value.</param>
|
/// <param name="value">The GUID value.</param>
|
||||||
public void WriteGuid(string name, Guid value)
|
public void WriteGuid(string name, Guid value)
|
||||||
{
|
{
|
||||||
WriteString(name, value.ToString());
|
WriteString(name, value.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON ObjectId element.
|
/// Writes a BSON ObjectId element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="value">The ObjectId value.</param>
|
/// <param name="value">The ObjectId value.</param>
|
||||||
public void WriteObjectId(string name, ObjectId value)
|
public void WriteObjectId(string name, ObjectId value)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.ObjectId, name);
|
WriteElementHeader(BsonType.ObjectId, name);
|
||||||
value.WriteTo(_buffer.Slice(_position, 12));
|
value.WriteTo(_buffer.Slice(Position, 12));
|
||||||
_position += 12;
|
Position += 12;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a BSON null element.
|
/// Writes a BSON null element.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
public void WriteNull(string name)
|
public void WriteNull(string name)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Null, name);
|
WriteElementHeader(BsonType.Null, name);
|
||||||
// No value to write for null
|
// No value to write for null
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes binary data
|
/// Writes binary data
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name.</param>
|
/// <param name="name">The field name.</param>
|
||||||
/// <param name="data">The binary payload.</param>
|
/// <param name="data">The binary payload.</param>
|
||||||
/// <param name="subtype">The BSON binary subtype.</param>
|
/// <param name="subtype">The BSON binary subtype.</param>
|
||||||
public void WriteBinary(string name, ReadOnlySpan<byte> data, byte subtype = 0)
|
public void WriteBinary(string name, ReadOnlySpan<byte> data, byte subtype = 0)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Binary, name);
|
WriteElementHeader(BsonType.Binary, name);
|
||||||
|
|
||||||
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(_position, 4), data.Length);
|
BinaryPrimitives.WriteInt32LittleEndian(_buffer.Slice(Position, 4), data.Length);
|
||||||
_position += 4;
|
Position += 4;
|
||||||
|
|
||||||
_buffer[_position] = subtype;
|
_buffer[Position] = subtype;
|
||||||
_position++;
|
Position++;
|
||||||
|
|
||||||
data.CopyTo(_buffer[_position..]);
|
data.CopyTo(_buffer[Position..]);
|
||||||
_position += data.Length;
|
Position += data.Length;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing a subdocument and returns the size position to patch later
|
/// Begins writing a subdocument and returns the size position to patch later
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name for the subdocument.</param>
|
/// <param name="name">The field name for the subdocument.</param>
|
||||||
public int BeginDocument(string name)
|
public int BeginDocument(string name)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Document, name);
|
WriteElementHeader(BsonType.Document, name);
|
||||||
return WriteDocumentSizePlaceholder();
|
return WriteDocumentSizePlaceholder();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing the root document and returns the size position to patch later
|
/// Begins writing the root document and returns the size position to patch later
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int BeginDocument()
|
public int BeginDocument()
|
||||||
{
|
{
|
||||||
return WriteDocumentSizePlaceholder();
|
return WriteDocumentSizePlaceholder();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ends the current document
|
/// Ends the current document
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder"/>.</param>
|
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder" />.</param>
|
||||||
public void EndDocument(int sizePosition)
|
public void EndDocument(int sizePosition)
|
||||||
{
|
{
|
||||||
WriteEndOfDocument();
|
WriteEndOfDocument();
|
||||||
PatchDocumentSize(sizePosition);
|
PatchDocumentSize(sizePosition);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins writing a BSON array and returns the size position to patch later
|
/// Begins writing a BSON array and returns the size position to patch later
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The field name for the array.</param>
|
/// <param name="name">The field name for the array.</param>
|
||||||
public int BeginArray(string name)
|
public int BeginArray(string name)
|
||||||
{
|
{
|
||||||
WriteElementHeader(BsonType.Array, name);
|
WriteElementHeader(BsonType.Array, name);
|
||||||
return WriteDocumentSizePlaceholder();
|
return WriteDocumentSizePlaceholder();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ends the current BSON array
|
/// Ends the current BSON array
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder"/>.</param>
|
/// <param name="sizePosition">The position returned by <see cref="WriteDocumentSizePlaceholder" />.</param>
|
||||||
public void EndArray(int sizePosition)
|
public void EndArray(int sizePosition)
|
||||||
{
|
{
|
||||||
WriteEndOfDocument();
|
WriteEndOfDocument();
|
||||||
PatchDocumentSize(sizePosition);
|
PatchDocumentSize(sizePosition);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,14 +1,11 @@
|
|||||||
using System;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Bson
|
[AttributeUsage(AttributeTargets.Property)]
|
||||||
|
public class BsonIdAttribute : Attribute
|
||||||
{
|
{
|
||||||
[AttributeUsage(AttributeTargets.Property)]
|
|
||||||
public class BsonIdAttribute : Attribute
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
[AttributeUsage(AttributeTargets.Property)]
|
|
||||||
public class BsonIgnoreAttribute : Attribute
|
|
||||||
{
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[AttributeUsage(AttributeTargets.Property)]
|
||||||
|
public class BsonIgnoreAttribute : Attribute
|
||||||
|
{
|
||||||
|
}
|
||||||
@@ -1,59 +1,56 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
|
|
||||||
public partial class BsonField
|
public class BsonField
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the field name.
|
/// Gets the field name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public required string Name { get; init; }
|
public required string Name { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the field BSON type.
|
/// Gets the field BSON type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public BsonType Type { get; init; }
|
public BsonType Type { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether the field is nullable.
|
/// Gets a value indicating whether the field is nullable.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsNullable { get; init; }
|
public bool IsNullable { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the nested schema when this field is a document.
|
/// Gets the nested schema when this field is a document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public BsonSchema? NestedSchema { get; init; }
|
public BsonSchema? NestedSchema { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the array item type when this field is an array.
|
/// Gets the array item type when this field is an array.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public BsonType? ArrayItemType { get; init; }
|
public BsonType? ArrayItemType { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes this field definition to BSON.
|
/// Writes this field definition to BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="writer">The BSON writer.</param>
|
/// <param name="writer">The BSON writer.</param>
|
||||||
public void ToBson(ref BsonSpanWriter writer)
|
public void ToBson(ref BsonSpanWriter writer)
|
||||||
{
|
{
|
||||||
var size = writer.BeginDocument();
|
int size = writer.BeginDocument();
|
||||||
writer.WriteString("n", Name);
|
writer.WriteString("n", Name);
|
||||||
writer.WriteInt32("t", (int)Type);
|
writer.WriteInt32("t", (int)Type);
|
||||||
writer.WriteBoolean("b", IsNullable);
|
writer.WriteBoolean("b", IsNullable);
|
||||||
|
|
||||||
if (NestedSchema != null)
|
if (NestedSchema != null)
|
||||||
{
|
{
|
||||||
writer.WriteElementHeader(BsonType.Document, "s");
|
writer.WriteElementHeader(BsonType.Document, "s");
|
||||||
NestedSchema.ToBson(ref writer);
|
NestedSchema.ToBson(ref writer);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ArrayItemType != null)
|
if (ArrayItemType != null) writer.WriteInt32("a", (int)ArrayItemType.Value);
|
||||||
{
|
|
||||||
writer.WriteInt32("a", (int)ArrayItemType.Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
writer.EndDocument(size);
|
writer.EndDocument(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a field definition from BSON.
|
/// Reads a field definition from BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reader">The BSON reader.</param>
|
/// <param name="reader">The BSON reader.</param>
|
||||||
/// <returns>The deserialized field.</returns>
|
/// <returns>The deserialized field.</returns>
|
||||||
@@ -61,59 +58,59 @@ public partial class BsonField
|
|||||||
{
|
{
|
||||||
reader.ReadInt32(); // Read doc size
|
reader.ReadInt32(); // Read doc size
|
||||||
|
|
||||||
string name = "";
|
var name = "";
|
||||||
BsonType type = BsonType.Null;
|
var type = BsonType.Null;
|
||||||
bool isNullable = false;
|
var isNullable = false;
|
||||||
BsonSchema? nestedSchema = null;
|
BsonSchema? nestedSchema = null;
|
||||||
BsonType? arrayItemType = null;
|
BsonType? arrayItemType = null;
|
||||||
|
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var btype = reader.ReadBsonType();
|
var btype = reader.ReadBsonType();
|
||||||
if (btype == BsonType.EndOfDocument) break;
|
if (btype == BsonType.EndOfDocument) break;
|
||||||
|
|
||||||
var key = reader.ReadElementHeader();
|
string key = reader.ReadElementHeader();
|
||||||
switch (key)
|
switch (key)
|
||||||
{
|
{
|
||||||
case "n": name = reader.ReadString(); break;
|
case "n": name = reader.ReadString(); break;
|
||||||
case "t": type = (BsonType)reader.ReadInt32(); break;
|
case "t": type = (BsonType)reader.ReadInt32(); break;
|
||||||
case "b": isNullable = reader.ReadBoolean(); break;
|
case "b": isNullable = reader.ReadBoolean(); break;
|
||||||
case "s": nestedSchema = BsonSchema.FromBson(ref reader); break;
|
case "s": nestedSchema = BsonSchema.FromBson(ref reader); break;
|
||||||
case "a": arrayItemType = (BsonType)reader.ReadInt32(); break;
|
case "a": arrayItemType = (BsonType)reader.ReadInt32(); break;
|
||||||
default: reader.SkipValue(btype); break;
|
default: reader.SkipValue(btype); break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return new BsonField
|
return new BsonField
|
||||||
{
|
{
|
||||||
Name = name,
|
Name = name,
|
||||||
Type = type,
|
Type = type,
|
||||||
IsNullable = isNullable,
|
IsNullable = isNullable,
|
||||||
NestedSchema = nestedSchema,
|
NestedSchema = nestedSchema,
|
||||||
ArrayItemType = arrayItemType
|
ArrayItemType = arrayItemType
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes a hash representing the field definition.
|
/// Computes a hash representing the field definition.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The computed hash value.</returns>
|
/// <returns>The computed hash value.</returns>
|
||||||
public long GetHash()
|
public long GetHash()
|
||||||
{
|
{
|
||||||
var hash = new HashCode();
|
var hash = new HashCode();
|
||||||
hash.Add(Name);
|
hash.Add(Name);
|
||||||
hash.Add((int)Type);
|
hash.Add((int)Type);
|
||||||
hash.Add(IsNullable);
|
hash.Add(IsNullable);
|
||||||
hash.Add(ArrayItemType);
|
hash.Add(ArrayItemType);
|
||||||
if (NestedSchema != null) hash.Add(NestedSchema.GetHash());
|
if (NestedSchema != null) hash.Add(NestedSchema.GetHash());
|
||||||
return hash.ToHashCode();
|
return hash.ToHashCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this field is equal to another field.
|
/// Determines whether this field is equal to another field.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The other field.</param>
|
/// <param name="other">The other field.</param>
|
||||||
/// <returns><see langword="true"/> if the fields are equal; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the fields are equal; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Equals(BsonField? other)
|
public bool Equals(BsonField? other)
|
||||||
{
|
{
|
||||||
if (other == null) return false;
|
if (other == null) return false;
|
||||||
@@ -121,8 +118,14 @@ public partial class BsonField
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override bool Equals(object? obj) => Equals(obj as BsonField);
|
public override bool Equals(object? obj)
|
||||||
|
{
|
||||||
|
return Equals(obj as BsonField);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override int GetHashCode() => (int)GetHash();
|
public override int GetHashCode()
|
||||||
}
|
{
|
||||||
|
return (int)GetHash();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,45 +1,46 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
namespace ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
|
|
||||||
public partial class BsonSchema
|
public class BsonSchema
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the schema title.
|
/// Gets or sets the schema title.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string? Title { get; set; }
|
public string? Title { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the schema version.
|
/// Gets or sets the schema version.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int? Version { get; set; }
|
public int? Version { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the schema fields.
|
/// Gets the schema fields.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<BsonField> Fields { get; } = new();
|
public List<BsonField> Fields { get; } = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Serializes this schema instance to BSON.
|
/// Serializes this schema instance to BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="writer">The BSON writer to write into.</param>
|
/// <param name="writer">The BSON writer to write into.</param>
|
||||||
public void ToBson(ref BsonSpanWriter writer)
|
public void ToBson(ref BsonSpanWriter writer)
|
||||||
{
|
{
|
||||||
var size = writer.BeginDocument();
|
int size = writer.BeginDocument();
|
||||||
if (Title != null) writer.WriteString("t", Title);
|
if (Title != null) writer.WriteString("t", Title);
|
||||||
if (Version != null) writer.WriteInt32("_v", Version.Value);
|
if (Version != null) writer.WriteInt32("_v", Version.Value);
|
||||||
|
|
||||||
var fieldsSize = writer.BeginArray("f");
|
int fieldsSize = writer.BeginArray("f");
|
||||||
for (int i = 0; i < Fields.Count; i++)
|
for (var i = 0; i < Fields.Count; i++)
|
||||||
{
|
{
|
||||||
writer.WriteElementHeader(BsonType.Document, i.ToString());
|
writer.WriteElementHeader(BsonType.Document, i.ToString());
|
||||||
Fields[i].ToBson(ref writer);
|
Fields[i].ToBson(ref writer);
|
||||||
}
|
}
|
||||||
|
|
||||||
writer.EndArray(fieldsSize);
|
writer.EndArray(fieldsSize);
|
||||||
|
|
||||||
writer.EndDocument(size);
|
writer.EndDocument(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deserializes a schema instance from BSON.
|
/// Deserializes a schema instance from BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reader">The BSON reader to read from.</param>
|
/// <param name="reader">The BSON reader to read from.</param>
|
||||||
/// <returns>The deserialized schema.</returns>
|
/// <returns>The deserialized schema.</returns>
|
||||||
@@ -47,55 +48,53 @@ public partial class BsonSchema
|
|||||||
{
|
{
|
||||||
reader.ReadInt32(); // Read doc size
|
reader.ReadInt32(); // Read doc size
|
||||||
|
|
||||||
var schema = new BsonSchema();
|
var schema = new BsonSchema();
|
||||||
|
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var btype = reader.ReadBsonType();
|
var btype = reader.ReadBsonType();
|
||||||
if (btype == BsonType.EndOfDocument) break;
|
if (btype == BsonType.EndOfDocument) break;
|
||||||
|
|
||||||
var key = reader.ReadElementHeader();
|
string key = reader.ReadElementHeader();
|
||||||
switch (key)
|
switch (key)
|
||||||
{
|
{
|
||||||
case "t": schema.Title = reader.ReadString(); break;
|
case "t": schema.Title = reader.ReadString(); break;
|
||||||
case "_v": schema.Version = reader.ReadInt32(); break;
|
case "_v": schema.Version = reader.ReadInt32(); break;
|
||||||
case "f":
|
case "f":
|
||||||
reader.ReadInt32(); // array size
|
reader.ReadInt32(); // array size
|
||||||
while (reader.Remaining > 1)
|
while (reader.Remaining > 1)
|
||||||
{
|
{
|
||||||
var itemType = reader.ReadBsonType();
|
var itemType = reader.ReadBsonType();
|
||||||
if (itemType == BsonType.EndOfDocument) break;
|
if (itemType == BsonType.EndOfDocument) break;
|
||||||
reader.ReadElementHeader(); // index
|
reader.ReadElementHeader(); // index
|
||||||
schema.Fields.Add(BsonField.FromBson(ref reader));
|
schema.Fields.Add(BsonField.FromBson(ref reader));
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
default: reader.SkipValue(btype); break;
|
break;
|
||||||
}
|
default: reader.SkipValue(btype); break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return schema;
|
return schema;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes a hash value for this schema based on its contents.
|
/// Computes a hash value for this schema based on its contents.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The computed hash value.</returns>
|
/// <returns>The computed hash value.</returns>
|
||||||
public long GetHash()
|
public long GetHash()
|
||||||
{
|
{
|
||||||
var hash = new HashCode();
|
var hash = new HashCode();
|
||||||
hash.Add(Title);
|
hash.Add(Title);
|
||||||
foreach (var field in Fields)
|
foreach (var field in Fields) hash.Add(field.GetHash());
|
||||||
{
|
|
||||||
hash.Add(field.GetHash());
|
|
||||||
}
|
|
||||||
return hash.ToHashCode();
|
return hash.ToHashCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this schema is equal to another schema.
|
/// Determines whether this schema is equal to another schema.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The schema to compare with.</param>
|
/// <param name="other">The schema to compare with.</param>
|
||||||
/// <returns><see langword="true"/> when schemas are equal; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> when schemas are equal; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Equals(BsonSchema? other)
|
public bool Equals(BsonSchema? other)
|
||||||
{
|
{
|
||||||
if (other == null) return false;
|
if (other == null) return false;
|
||||||
@@ -103,27 +102,29 @@ public partial class BsonSchema
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override bool Equals(object? obj) => Equals(obj as BsonSchema);
|
public override bool Equals(object? obj)
|
||||||
|
{
|
||||||
|
return Equals(obj as BsonSchema);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override int GetHashCode() => (int)GetHash();
|
public override int GetHashCode()
|
||||||
|
{
|
||||||
|
return (int)GetHash();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Enumerates all field keys in this schema, including nested schema keys.
|
/// Enumerates all field keys in this schema, including nested schema keys.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>An enumerable of field keys.</returns>
|
/// <returns>An enumerable of field keys.</returns>
|
||||||
public IEnumerable<string> GetAllKeys()
|
public IEnumerable<string> GetAllKeys()
|
||||||
{
|
{
|
||||||
foreach (var field in Fields)
|
foreach (var field in Fields)
|
||||||
{
|
{
|
||||||
yield return field.Name;
|
yield return field.Name;
|
||||||
if (field.NestedSchema != null)
|
if (field.NestedSchema != null)
|
||||||
{
|
foreach (string nestedKey in field.NestedSchema.GetAllKeys())
|
||||||
foreach (var nestedKey in field.NestedSchema.GetAllKeys())
|
yield return nestedKey;
|
||||||
{
|
}
|
||||||
yield return nestedKey;
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,11 +1,10 @@
|
|||||||
using System;
|
|
||||||
using System.Runtime.InteropServices;
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Bson;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// 12-byte ObjectId compatible with MongoDB ObjectId.
|
/// 12-byte ObjectId compatible with MongoDB ObjectId.
|
||||||
/// Implemented as readonly struct for zero allocation.
|
/// Implemented as readonly struct for zero allocation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[StructLayout(LayoutKind.Explicit, Size = 12)]
|
[StructLayout(LayoutKind.Explicit, Size = 12)]
|
||||||
public readonly struct ObjectId : IEquatable<ObjectId>
|
public readonly struct ObjectId : IEquatable<ObjectId>
|
||||||
@@ -14,20 +13,20 @@ public readonly struct ObjectId : IEquatable<ObjectId>
|
|||||||
[FieldOffset(4)] private readonly long _randomAndCounter;
|
[FieldOffset(4)] private readonly long _randomAndCounter;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Empty ObjectId (all zeros)
|
/// Empty ObjectId (all zeros)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static readonly ObjectId Empty = new ObjectId(0, 0);
|
public static readonly ObjectId Empty = new(0, 0);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Maximum ObjectId (all 0xFF bytes) - useful for range queries
|
/// Maximum ObjectId (all 0xFF bytes) - useful for range queries
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static readonly ObjectId MaxValue = new ObjectId(int.MaxValue, long.MaxValue);
|
public static readonly ObjectId MaxValue = new(int.MaxValue, long.MaxValue);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="ObjectId"/> struct from raw bytes.
|
/// Initializes a new instance of the <see cref="ObjectId" /> struct from raw bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="bytes">The 12-byte ObjectId value.</param>
|
/// <param name="bytes">The 12-byte ObjectId value.</param>
|
||||||
public ObjectId(ReadOnlySpan<byte> bytes)
|
public ObjectId(ReadOnlySpan<byte> bytes)
|
||||||
{
|
{
|
||||||
if (bytes.Length != 12)
|
if (bytes.Length != 12)
|
||||||
throw new ArgumentException("ObjectId must be exactly 12 bytes", nameof(bytes));
|
throw new ArgumentException("ObjectId must be exactly 12 bytes", nameof(bytes));
|
||||||
@@ -36,32 +35,32 @@ public readonly struct ObjectId : IEquatable<ObjectId>
|
|||||||
_randomAndCounter = BitConverter.ToInt64(bytes[4..12]);
|
_randomAndCounter = BitConverter.ToInt64(bytes[4..12]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="ObjectId"/> struct from its components.
|
/// Initializes a new instance of the <see cref="ObjectId" /> struct from its components.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="timestamp">The Unix timestamp portion.</param>
|
/// <param name="timestamp">The Unix timestamp portion.</param>
|
||||||
/// <param name="randomAndCounter">The random and counter portion.</param>
|
/// <param name="randomAndCounter">The random and counter portion.</param>
|
||||||
public ObjectId(int timestamp, long randomAndCounter)
|
public ObjectId(int timestamp, long randomAndCounter)
|
||||||
{
|
{
|
||||||
_timestamp = timestamp;
|
_timestamp = timestamp;
|
||||||
_randomAndCounter = randomAndCounter;
|
_randomAndCounter = randomAndCounter;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new ObjectId with current timestamp
|
/// Creates a new ObjectId with current timestamp
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static ObjectId NewObjectId()
|
public static ObjectId NewObjectId()
|
||||||
{
|
{
|
||||||
var timestamp = (int)DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
var timestamp = (int)DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
||||||
var random = Random.Shared.NextInt64();
|
long random = Random.Shared.NextInt64();
|
||||||
return new ObjectId(timestamp, random);
|
return new ObjectId(timestamp, random);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes the ObjectId to the destination span (must be 12 bytes)
|
/// Writes the ObjectId to the destination span (must be 12 bytes)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span to write into.</param>
|
/// <param name="destination">The destination span to write into.</param>
|
||||||
public void WriteTo(Span<byte> destination)
|
public void WriteTo(Span<byte> destination)
|
||||||
{
|
{
|
||||||
if (destination.Length < 12)
|
if (destination.Length < 12)
|
||||||
throw new ArgumentException("Destination must be at least 12 bytes", nameof(destination));
|
throw new ArgumentException("Destination must be at least 12 bytes", nameof(destination));
|
||||||
@@ -71,7 +70,7 @@ public readonly struct ObjectId : IEquatable<ObjectId>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts ObjectId to byte array
|
/// Converts ObjectId to byte array
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public byte[] ToByteArray()
|
public byte[] ToByteArray()
|
||||||
{
|
{
|
||||||
@@ -81,32 +80,47 @@ public readonly struct ObjectId : IEquatable<ObjectId>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets timestamp portion as UTC DateTime
|
/// Gets timestamp portion as UTC DateTime
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(_timestamp).UtcDateTime;
|
public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(_timestamp).UtcDateTime;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this instance and another <see cref="ObjectId"/> have the same value.
|
/// Determines whether this instance and another <see cref="ObjectId" /> have the same value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The object to compare with this instance.</param>
|
/// <param name="other">The object to compare with this instance.</param>
|
||||||
/// <returns><see langword="true"/> if the values are equal; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the values are equal; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Equals(ObjectId other) =>
|
public bool Equals(ObjectId other)
|
||||||
_timestamp == other._timestamp && _randomAndCounter == other._randomAndCounter;
|
{
|
||||||
|
return _timestamp == other._timestamp && _randomAndCounter == other._randomAndCounter;
|
||||||
/// <inheritdoc />
|
}
|
||||||
public override bool Equals(object? obj) => obj is ObjectId other && Equals(other);
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
public override int GetHashCode() => HashCode.Combine(_timestamp, _randomAndCounter);
|
|
||||||
|
|
||||||
public static bool operator ==(ObjectId left, ObjectId right) => left.Equals(right);
|
/// <inheritdoc />
|
||||||
public static bool operator !=(ObjectId left, ObjectId right) => !left.Equals(right);
|
public override bool Equals(object? obj)
|
||||||
|
{
|
||||||
|
return obj is ObjectId other && Equals(other);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string ToString()
|
public override int GetHashCode()
|
||||||
|
{
|
||||||
|
return HashCode.Combine(_timestamp, _randomAndCounter);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator ==(ObjectId left, ObjectId right)
|
||||||
|
{
|
||||||
|
return left.Equals(right);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator !=(ObjectId left, ObjectId right)
|
||||||
|
{
|
||||||
|
return !left.Equals(right);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override string ToString()
|
||||||
{
|
{
|
||||||
Span<byte> bytes = stackalloc byte[12];
|
Span<byte> bytes = stackalloc byte[12];
|
||||||
WriteTo(bytes);
|
WriteTo(bytes);
|
||||||
return Convert.ToHexString(bytes).ToLowerInvariant();
|
return Convert.ToHexString(bytes).ToLowerInvariant();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,28 +1,28 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<AssemblyName>ZB.MOM.WW.CBDD.Bson</AssemblyName>
|
<AssemblyName>ZB.MOM.WW.CBDD.Bson</AssemblyName>
|
||||||
<RootNamespace>ZB.MOM.WW.CBDD.Bson</RootNamespace>
|
<RootNamespace>ZB.MOM.WW.CBDD.Bson</RootNamespace>
|
||||||
<LangVersion>latest</LangVersion>
|
<LangVersion>latest</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||||
|
|
||||||
<PackageId>ZB.MOM.WW.CBDD.Bson</PackageId>
|
|
||||||
<Version>1.3.1</Version>
|
|
||||||
<Authors>CBDD Team</Authors>
|
|
||||||
<Description>BSON Serialization Library for High-Performance Database Engine</Description>
|
|
||||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
|
||||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
|
||||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
|
||||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
<PackageId>ZB.MOM.WW.CBDD.Bson</PackageId>
|
||||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
<Version>1.3.1</Version>
|
||||||
</ItemGroup>
|
<Authors>CBDD Team</Authors>
|
||||||
|
<Description>BSON Serialization Library for High-Performance Database Engine</Description>
|
||||||
|
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||||
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
|
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||||
|
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||||
|
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="..\..\README.md" Pack="true" PackagePath="\"/>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,23 +1,21 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Channels;
|
using System.Threading.Channels;
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
|
|
||||||
internal sealed class ChangeStreamDispatcher : IDisposable
|
internal sealed class ChangeStreamDispatcher : IDisposable
|
||||||
{
|
{
|
||||||
private readonly Channel<InternalChangeEvent> _channel;
|
private readonly Channel<InternalChangeEvent> _channel;
|
||||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>> _subscriptions = new();
|
|
||||||
private readonly ConcurrentDictionary<string, int> _payloadWatcherCounts = new();
|
|
||||||
private readonly CancellationTokenSource _cts = new();
|
private readonly CancellationTokenSource _cts = new();
|
||||||
|
private readonly ConcurrentDictionary<string, int> _payloadWatcherCounts = new();
|
||||||
|
|
||||||
/// <summary>
|
private readonly ConcurrentDictionary<string, ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>>
|
||||||
/// Initializes a new change stream dispatcher.
|
_subscriptions = new();
|
||||||
/// </summary>
|
|
||||||
public ChangeStreamDispatcher()
|
/// <summary>
|
||||||
|
/// Initializes a new change stream dispatcher.
|
||||||
|
/// </summary>
|
||||||
|
public ChangeStreamDispatcher()
|
||||||
{
|
{
|
||||||
_channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
_channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
||||||
{
|
{
|
||||||
@@ -28,50 +26,57 @@ internal sealed class ChangeStreamDispatcher : IDisposable
|
|||||||
Task.Run(ProcessEventsAsync);
|
Task.Run(ProcessEventsAsync);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Publishes a change event to subscribers.
|
/// Releases dispatcher resources.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="change">The change event to publish.</param>
|
public void Dispose()
|
||||||
public void Publish(InternalChangeEvent change)
|
{
|
||||||
|
_cts.Cancel();
|
||||||
|
_cts.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Publishes a change event to subscribers.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="change">The change event to publish.</param>
|
||||||
|
public void Publish(InternalChangeEvent change)
|
||||||
{
|
{
|
||||||
_channel.Writer.TryWrite(change);
|
_channel.Writer.TryWrite(change);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether a collection has subscribers that require payloads.
|
/// Determines whether a collection has subscribers that require payloads.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="collectionName">The collection name.</param>
|
/// <param name="collectionName">The collection name.</param>
|
||||||
/// <returns><see langword="true"/> if payload watchers exist; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if payload watchers exist; otherwise, <see langword="false" />.</returns>
|
||||||
public bool HasPayloadWatchers(string collectionName)
|
public bool HasPayloadWatchers(string collectionName)
|
||||||
{
|
{
|
||||||
return _payloadWatcherCounts.TryGetValue(collectionName, out var count) && count > 0;
|
return _payloadWatcherCounts.TryGetValue(collectionName, out int count) && count > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether a collection has any subscribers.
|
/// Determines whether a collection has any subscribers.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="collectionName">The collection name.</param>
|
/// <param name="collectionName">The collection name.</param>
|
||||||
/// <returns><see langword="true"/> if subscribers exist; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if subscribers exist; otherwise, <see langword="false" />.</returns>
|
||||||
public bool HasAnyWatchers(string collectionName)
|
public bool HasAnyWatchers(string collectionName)
|
||||||
{
|
{
|
||||||
return _subscriptions.TryGetValue(collectionName, out var subs) && !subs.IsEmpty;
|
return _subscriptions.TryGetValue(collectionName, out var subs) && !subs.IsEmpty;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Subscribes a channel writer to collection change events.
|
/// Subscribes a channel writer to collection change events.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="collectionName">The collection name to subscribe to.</param>
|
/// <param name="collectionName">The collection name to subscribe to.</param>
|
||||||
/// <param name="capturePayload">Whether this subscriber requires event payloads.</param>
|
/// <param name="capturePayload">Whether this subscriber requires event payloads.</param>
|
||||||
/// <param name="writer">The destination channel writer.</param>
|
/// <param name="writer">The destination channel writer.</param>
|
||||||
/// <returns>An <see cref="IDisposable"/> that removes the subscription when disposed.</returns>
|
/// <returns>An <see cref="IDisposable" /> that removes the subscription when disposed.</returns>
|
||||||
public IDisposable Subscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
public IDisposable Subscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
||||||
{
|
{
|
||||||
if (capturePayload)
|
if (capturePayload) _payloadWatcherCounts.AddOrUpdate(collectionName, 1, (_, count) => count + 1);
|
||||||
{
|
|
||||||
_payloadWatcherCounts.AddOrUpdate(collectionName, 1, (_, count) => count + 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
var collectionSubs = _subscriptions.GetOrAdd(collectionName, _ => new ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>());
|
var collectionSubs = _subscriptions.GetOrAdd(collectionName,
|
||||||
|
_ => new ConcurrentDictionary<ChannelWriter<InternalChangeEvent>, byte>());
|
||||||
collectionSubs.TryAdd(writer, 0);
|
collectionSubs.TryAdd(writer, 0);
|
||||||
|
|
||||||
return new Subscription(() => Unsubscribe(collectionName, capturePayload, writer));
|
return new Subscription(() => Unsubscribe(collectionName, capturePayload, writer));
|
||||||
@@ -79,15 +84,9 @@ internal sealed class ChangeStreamDispatcher : IDisposable
|
|||||||
|
|
||||||
private void Unsubscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
private void Unsubscribe(string collectionName, bool capturePayload, ChannelWriter<InternalChangeEvent> writer)
|
||||||
{
|
{
|
||||||
if (_subscriptions.TryGetValue(collectionName, out var collectionSubs))
|
if (_subscriptions.TryGetValue(collectionName, out var collectionSubs)) collectionSubs.TryRemove(writer, out _);
|
||||||
{
|
|
||||||
collectionSubs.TryRemove(writer, out _);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (capturePayload)
|
if (capturePayload) _payloadWatcherCounts.AddOrUpdate(collectionName, 0, (_, count) => Math.Max(0, count - 1));
|
||||||
{
|
|
||||||
_payloadWatcherCounts.AddOrUpdate(collectionName, 0, (_, count) => Math.Max(0, count - 1));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ProcessEventsAsync()
|
private async Task ProcessEventsAsync()
|
||||||
@@ -96,60 +95,45 @@ internal sealed class ChangeStreamDispatcher : IDisposable
|
|||||||
{
|
{
|
||||||
var reader = _channel.Reader;
|
var reader = _channel.Reader;
|
||||||
while (await reader.WaitToReadAsync(_cts.Token))
|
while (await reader.WaitToReadAsync(_cts.Token))
|
||||||
{
|
while (reader.TryRead(out var @event))
|
||||||
while (reader.TryRead(out var @event))
|
if (_subscriptions.TryGetValue(@event.CollectionName, out var collectionSubs))
|
||||||
{
|
foreach (var writer in collectionSubs.Keys)
|
||||||
if (_subscriptions.TryGetValue(@event.CollectionName, out var collectionSubs))
|
// Optimized fan-out: non-blocking TryWrite.
|
||||||
{
|
// If a subscriber channel is full (unlikely with Unbounded),
|
||||||
foreach (var writer in collectionSubs.Keys)
|
// we skip or drop. Usually, subscribers will also use Unbounded.
|
||||||
{
|
writer.TryWrite(@event);
|
||||||
// Optimized fan-out: non-blocking TryWrite.
|
}
|
||||||
// If a subscriber channel is full (unlikely with Unbounded),
|
catch (OperationCanceledException)
|
||||||
// we skip or drop. Usually, subscribers will also use Unbounded.
|
{
|
||||||
writer.TryWrite(@event);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) { }
|
|
||||||
catch (Exception)
|
catch (Exception)
|
||||||
{
|
{
|
||||||
// Internal error logging could go here
|
// Internal error logging could go here
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Releases dispatcher resources.
|
|
||||||
/// </summary>
|
|
||||||
public void Dispose()
|
|
||||||
{
|
|
||||||
_cts.Cancel();
|
|
||||||
_cts.Dispose();
|
|
||||||
}
|
|
||||||
|
|
||||||
private sealed class Subscription : IDisposable
|
private sealed class Subscription : IDisposable
|
||||||
{
|
{
|
||||||
private readonly Action _onDispose;
|
private readonly Action _onDispose;
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new subscription token.
|
/// Initializes a new subscription token.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="onDispose">Callback executed when the subscription is disposed.</param>
|
/// <param name="onDispose">Callback executed when the subscription is disposed.</param>
|
||||||
public Subscription(Action onDispose)
|
public Subscription(Action onDispose)
|
||||||
{
|
{
|
||||||
_onDispose = onDispose;
|
_onDispose = onDispose;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Disposes the subscription and unregisters the subscriber.
|
/// Disposes the subscription and unregisters the subscriber.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
if (_disposed) return;
|
if (_disposed) return;
|
||||||
_onDispose();
|
_onDispose();
|
||||||
_disposed = true;
|
_disposed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,76 +1,75 @@
|
|||||||
using System;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// A generic, immutable struct representing a data change in a collection.
|
||||||
/// A generic, immutable struct representing a data change in a collection.
|
/// </summary>
|
||||||
/// </summary>
|
|
||||||
public readonly struct ChangeStreamEvent<TId, T> where T : class
|
public readonly struct ChangeStreamEvent<TId, T> where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the UTC timestamp when the change was recorded.
|
/// Gets the UTC timestamp when the change was recorded.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long Timestamp { get; init; }
|
public long Timestamp { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the transaction identifier that produced the change.
|
/// Gets the transaction identifier that produced the change.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ulong TransactionId { get; init; }
|
public ulong TransactionId { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection name where the change occurred.
|
/// Gets the collection name where the change occurred.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string CollectionName { get; init; }
|
public string CollectionName { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the operation type associated with the change.
|
/// Gets the operation type associated with the change.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public OperationType Type { get; init; }
|
public OperationType Type { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the changed document identifier.
|
/// Gets the changed document identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public TId DocumentId { get; init; }
|
public TId DocumentId { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// The deserialized entity. Null if capturePayload was false during Watch().
|
/// The deserialized entity. Null if capturePayload was false during Watch().
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public T? Entity { get; init; }
|
public T? Entity { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Low-level event structure used internally to transport changes before deserialization.
|
/// Low-level event structure used internally to transport changes before deserialization.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal readonly struct InternalChangeEvent
|
internal readonly struct InternalChangeEvent
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the UTC timestamp when the change was recorded.
|
/// Gets the UTC timestamp when the change was recorded.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long Timestamp { get; init; }
|
public long Timestamp { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the transaction identifier that produced the change.
|
/// Gets the transaction identifier that produced the change.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ulong TransactionId { get; init; }
|
public ulong TransactionId { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection name where the change occurred.
|
/// Gets the collection name where the change occurred.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string CollectionName { get; init; }
|
public string CollectionName { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the operation type associated with the change.
|
/// Gets the operation type associated with the change.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public OperationType Type { get; init; }
|
public OperationType Type { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Raw BSON of the Document ID.
|
/// Raw BSON of the Document ID.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ReadOnlyMemory<byte> IdBytes { get; init; }
|
public ReadOnlyMemory<byte> IdBytes { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Raw BSON of the Entity. Null if payload not captured.
|
/// Raw BSON of the Entity. Null if payload not captured.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ReadOnlyMemory<byte>? PayloadBytes { get; init; }
|
public ReadOnlyMemory<byte>? PayloadBytes { get; init; }
|
||||||
}
|
}
|
||||||
@@ -1,49 +1,45 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Threading.Channels;
|
using System.Threading.Channels;
|
||||||
using System.Threading.Tasks;
|
|
||||||
using System.Threading;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
|
|
||||||
internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamEvent<TId, T>> where T : class
|
internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamEvent<TId, T>> where T : class
|
||||||
{
|
{
|
||||||
private readonly ChangeStreamDispatcher _dispatcher;
|
|
||||||
private readonly string _collectionName;
|
|
||||||
private readonly bool _capturePayload;
|
private readonly bool _capturePayload;
|
||||||
private readonly IDocumentMapper<TId, T> _mapper;
|
private readonly string _collectionName;
|
||||||
|
private readonly ChangeStreamDispatcher _dispatcher;
|
||||||
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
||||||
|
private readonly IDocumentMapper<TId, T> _mapper;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new observable wrapper for collection change events.
|
/// Initializes a new observable wrapper for collection change events.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="dispatcher">The dispatcher producing internal change events.</param>
|
/// <param name="dispatcher">The dispatcher producing internal change events.</param>
|
||||||
/// <param name="collectionName">The collection to subscribe to.</param>
|
/// <param name="collectionName">The collection to subscribe to.</param>
|
||||||
/// <param name="capturePayload">Whether full entity payloads should be included.</param>
|
/// <param name="capturePayload">Whether full entity payloads should be included.</param>
|
||||||
/// <param name="mapper">The document mapper used for ID and payload deserialization.</param>
|
/// <param name="mapper">The document mapper used for ID and payload deserialization.</param>
|
||||||
/// <param name="keyReverseMap">The key reverse map used by BSON readers.</param>
|
/// <param name="keyReverseMap">The key reverse map used by BSON readers.</param>
|
||||||
public ChangeStreamObservable(
|
public ChangeStreamObservable(
|
||||||
ChangeStreamDispatcher dispatcher,
|
ChangeStreamDispatcher dispatcher,
|
||||||
string collectionName,
|
string collectionName,
|
||||||
bool capturePayload,
|
bool capturePayload,
|
||||||
IDocumentMapper<TId, T> mapper,
|
IDocumentMapper<TId, T> mapper,
|
||||||
ConcurrentDictionary<ushort, string> keyReverseMap)
|
ConcurrentDictionary<ushort, string> keyReverseMap)
|
||||||
{
|
{
|
||||||
_dispatcher = dispatcher;
|
_dispatcher = dispatcher;
|
||||||
_collectionName = collectionName;
|
_collectionName = collectionName;
|
||||||
_capturePayload = capturePayload;
|
_capturePayload = capturePayload;
|
||||||
_mapper = mapper;
|
_mapper = mapper;
|
||||||
_keyReverseMap = keyReverseMap;
|
_keyReverseMap = keyReverseMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public IDisposable Subscribe(IObserver<ChangeStreamEvent<TId, T>> observer)
|
public IDisposable Subscribe(IObserver<ChangeStreamEvent<TId, T>> observer)
|
||||||
{
|
{
|
||||||
if (observer == null) throw new ArgumentNullException(nameof(observer));
|
if (observer == null) throw new ArgumentNullException(nameof(observer));
|
||||||
|
|
||||||
var cts = new CancellationTokenSource();
|
var cts = new CancellationTokenSource();
|
||||||
var channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
var channel = Channel.CreateUnbounded<InternalChangeEvent>(new UnboundedChannelOptions
|
||||||
@@ -60,46 +56,43 @@ internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamE
|
|||||||
return new CompositeDisposable(dispatcherSubscription, cts, channel.Writer, bridgeTask);
|
return new CompositeDisposable(dispatcherSubscription, cts, channel.Writer, bridgeTask);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task BridgeChannelToObserverAsync(ChannelReader<InternalChangeEvent> reader, IObserver<ChangeStreamEvent<TId, T>> observer, CancellationToken ct)
|
private async Task BridgeChannelToObserverAsync(ChannelReader<InternalChangeEvent> reader,
|
||||||
|
IObserver<ChangeStreamEvent<TId, T>> observer, CancellationToken ct)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
while (await reader.WaitToReadAsync(ct))
|
while (await reader.WaitToReadAsync(ct))
|
||||||
{
|
while (reader.TryRead(out var internalEvent))
|
||||||
while (reader.TryRead(out var internalEvent))
|
try
|
||||||
{
|
{
|
||||||
try
|
// Deserializza ID
|
||||||
{
|
var eventId = _mapper.FromIndexKey(new IndexKey(internalEvent.IdBytes.ToArray()));
|
||||||
// Deserializza ID
|
|
||||||
var eventId = _mapper.FromIndexKey(new IndexKey(internalEvent.IdBytes.ToArray()));
|
|
||||||
|
|
||||||
// Deserializza Payload (se presente)
|
|
||||||
T? entity = default;
|
|
||||||
if (internalEvent.PayloadBytes.HasValue)
|
|
||||||
{
|
|
||||||
entity = _mapper.Deserialize(new BsonSpanReader(internalEvent.PayloadBytes.Value.Span, _keyReverseMap));
|
|
||||||
}
|
|
||||||
|
|
||||||
var externalEvent = new ChangeStreamEvent<TId, T>
|
// Deserializza Payload (se presente)
|
||||||
{
|
T? entity = default;
|
||||||
Timestamp = internalEvent.Timestamp,
|
if (internalEvent.PayloadBytes.HasValue)
|
||||||
TransactionId = internalEvent.TransactionId,
|
entity = _mapper.Deserialize(new BsonSpanReader(internalEvent.PayloadBytes.Value.Span,
|
||||||
CollectionName = internalEvent.CollectionName,
|
_keyReverseMap));
|
||||||
Type = internalEvent.Type,
|
|
||||||
DocumentId = eventId,
|
|
||||||
Entity = entity
|
|
||||||
};
|
|
||||||
|
|
||||||
observer.OnNext(externalEvent);
|
var externalEvent = new ChangeStreamEvent<TId, T>
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
{
|
||||||
// In case of deserialization error, we notify and continue if possible
|
Timestamp = internalEvent.Timestamp,
|
||||||
// Or we can stop the observer.
|
TransactionId = internalEvent.TransactionId,
|
||||||
observer.OnError(ex);
|
CollectionName = internalEvent.CollectionName,
|
||||||
}
|
Type = internalEvent.Type,
|
||||||
|
DocumentId = eventId,
|
||||||
|
Entity = entity
|
||||||
|
};
|
||||||
|
|
||||||
|
observer.OnNext(externalEvent);
|
||||||
}
|
}
|
||||||
}
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// In case of deserialization error, we notify and continue if possible
|
||||||
|
// Or we can stop the observer.
|
||||||
|
observer.OnError(ex);
|
||||||
|
}
|
||||||
|
|
||||||
observer.OnCompleted();
|
observer.OnCompleted();
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException)
|
catch (OperationCanceledException)
|
||||||
@@ -112,33 +105,34 @@ internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamE
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class CompositeDisposable : IDisposable
|
private sealed class CompositeDisposable : IDisposable
|
||||||
{
|
{
|
||||||
private readonly IDisposable _dispatcherSubscription;
|
|
||||||
private readonly CancellationTokenSource _cts;
|
|
||||||
private readonly ChannelWriter<InternalChangeEvent> _writer;
|
|
||||||
private readonly Task _bridgeTask;
|
private readonly Task _bridgeTask;
|
||||||
|
private readonly CancellationTokenSource _cts;
|
||||||
|
private readonly IDisposable _dispatcherSubscription;
|
||||||
|
private readonly ChannelWriter<InternalChangeEvent> _writer;
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new disposable wrapper for change stream resources.
|
/// Initializes a new disposable wrapper for change stream resources.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="dispatcherSubscription">The dispatcher subscription handle.</param>
|
/// <param name="dispatcherSubscription">The dispatcher subscription handle.</param>
|
||||||
/// <param name="cts">The cancellation source controlling the bridge task.</param>
|
/// <param name="cts">The cancellation source controlling the bridge task.</param>
|
||||||
/// <param name="writer">The channel writer for internal change events.</param>
|
/// <param name="writer">The channel writer for internal change events.</param>
|
||||||
/// <param name="bridgeTask">The running bridge task.</param>
|
/// <param name="bridgeTask">The running bridge task.</param>
|
||||||
public CompositeDisposable(IDisposable dispatcherSubscription, CancellationTokenSource cts, ChannelWriter<InternalChangeEvent> writer, Task bridgeTask)
|
public CompositeDisposable(IDisposable dispatcherSubscription, CancellationTokenSource cts,
|
||||||
{
|
ChannelWriter<InternalChangeEvent> writer, Task bridgeTask)
|
||||||
_dispatcherSubscription = dispatcherSubscription;
|
{
|
||||||
_cts = cts;
|
_dispatcherSubscription = dispatcherSubscription;
|
||||||
_writer = writer;
|
_cts = cts;
|
||||||
_bridgeTask = bridgeTask;
|
_writer = writer;
|
||||||
}
|
_bridgeTask = bridgeTask;
|
||||||
|
}
|
||||||
/// <inheritdoc />
|
|
||||||
public void Dispose()
|
/// <inheritdoc />
|
||||||
{
|
public void Dispose()
|
||||||
if (_disposed) return;
|
{
|
||||||
|
if (_disposed) return;
|
||||||
_disposed = true;
|
_disposed = true;
|
||||||
|
|
||||||
_dispatcherSubscription.Dispose();
|
_dispatcherSubscription.Dispose();
|
||||||
@@ -147,4 +141,4 @@ internal sealed class ChangeStreamObservable<TId, T> : IObservable<ChangeStreamE
|
|||||||
_cts.Dispose();
|
_cts.Dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,25 @@
|
|||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
namespace ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Handles CDC watch/notify behavior for a single collection.
|
/// Handles CDC watch/notify behavior for a single collection.
|
||||||
/// Extracted from DocumentCollection to keep storage/query concerns separated from event plumbing.
|
/// Extracted from DocumentCollection to keep storage/query concerns separated from event plumbing.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TId">Document identifier type.</typeparam>
|
/// <typeparam name="TId">Document identifier type.</typeparam>
|
||||||
/// <typeparam name="T">Document type.</typeparam>
|
/// <typeparam name="T">Document type.</typeparam>
|
||||||
internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
||||||
{
|
{
|
||||||
private readonly ITransactionHolder _transactionHolder;
|
|
||||||
private readonly string _collectionName;
|
private readonly string _collectionName;
|
||||||
private readonly IDocumentMapper<TId, T> _mapper;
|
|
||||||
private readonly ChangeStreamDispatcher? _dispatcher;
|
private readonly ChangeStreamDispatcher? _dispatcher;
|
||||||
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
private readonly ConcurrentDictionary<ushort, string> _keyReverseMap;
|
||||||
|
private readonly IDocumentMapper<TId, T> _mapper;
|
||||||
|
private readonly ITransactionHolder _transactionHolder;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="CollectionCdcPublisher"/> class.
|
/// Initializes a new instance of the <see cref="CollectionCdcPublisher" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionHolder">The transaction holder.</param>
|
/// <param name="transactionHolder">The transaction holder.</param>
|
||||||
/// <param name="collectionName">The collection name.</param>
|
/// <param name="collectionName">The collection name.</param>
|
||||||
@@ -42,7 +41,7 @@ internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes Watch.
|
/// Executes Watch.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="capturePayload">Whether to include payload data.</param>
|
/// <param name="capturePayload">Whether to include payload data.</param>
|
||||||
public IObservable<ChangeStreamEvent<TId, T>> Watch(bool capturePayload = false)
|
public IObservable<ChangeStreamEvent<TId, T>> Watch(bool capturePayload = false)
|
||||||
@@ -59,7 +58,7 @@ internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes Notify.
|
/// Executes Notify.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="type">The operation type.</param>
|
/// <param name="type">The operation type.</param>
|
||||||
/// <param name="id">The document identifier.</param>
|
/// <param name="id">The document identifier.</param>
|
||||||
@@ -74,15 +73,11 @@ internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
ReadOnlyMemory<byte>? payload = null;
|
ReadOnlyMemory<byte>? payload = null;
|
||||||
if (!docData.IsEmpty && _dispatcher.HasPayloadWatchers(_collectionName))
|
if (!docData.IsEmpty && _dispatcher.HasPayloadWatchers(_collectionName)) payload = docData.ToArray();
|
||||||
{
|
|
||||||
payload = docData.ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
var idBytes = _mapper.ToIndexKey(id).Data.ToArray();
|
byte[] idBytes = _mapper.ToIndexKey(id).Data.ToArray();
|
||||||
|
|
||||||
if (transaction is Transaction t)
|
if (transaction is Transaction t)
|
||||||
{
|
|
||||||
t.AddChange(new InternalChangeEvent
|
t.AddChange(new InternalChangeEvent
|
||||||
{
|
{
|
||||||
Timestamp = DateTime.UtcNow.Ticks,
|
Timestamp = DateTime.UtcNow.Ticks,
|
||||||
@@ -92,6 +87,5 @@ internal sealed class CollectionCdcPublisher<TId, T> where T : class
|
|||||||
IdBytes = idBytes,
|
IdBytes = idBytes,
|
||||||
PayloadBytes = payload
|
PayloadBytes = payload
|
||||||
});
|
});
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,25 +1,21 @@
|
|||||||
using System;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using System.Buffers;
|
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
using System.Linq;
|
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using System.Collections.Generic;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
/// <summary>
|
||||||
|
/// Base class for custom mappers that provides bidirectional IndexKey mapping for standard types.
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
/// </summary>
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Base class for custom mappers that provides bidirectional IndexKey mapping for standard types.
|
|
||||||
/// </summary>
|
|
||||||
public abstract class DocumentMapperBase<TId, T> : IDocumentMapper<TId, T> where T : class
|
public abstract class DocumentMapperBase<TId, T> : IDocumentMapper<TId, T> where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the target collection name for the mapped entity type.
|
/// Gets the target collection name for the mapped entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract string CollectionName { get; }
|
public abstract string CollectionName { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Serializes an entity instance into BSON.
|
/// Serializes an entity instance into BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity to serialize.</param>
|
/// <param name="entity">The entity to serialize.</param>
|
||||||
/// <param name="writer">The BSON writer to write into.</param>
|
/// <param name="writer">The BSON writer to write into.</param>
|
||||||
@@ -27,96 +23,129 @@ public abstract class DocumentMapperBase<TId, T> : IDocumentMapper<TId, T> where
|
|||||||
public abstract int Serialize(T entity, BsonSpanWriter writer);
|
public abstract int Serialize(T entity, BsonSpanWriter writer);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deserializes an entity instance from BSON.
|
/// Deserializes an entity instance from BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reader">The BSON reader to read from.</param>
|
/// <param name="reader">The BSON reader to read from.</param>
|
||||||
/// <returns>The deserialized entity.</returns>
|
/// <returns>The deserialized entity.</returns>
|
||||||
public abstract T Deserialize(BsonSpanReader reader);
|
public abstract T Deserialize(BsonSpanReader reader);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the identifier value from an entity.
|
/// Gets the identifier value from an entity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity to read the identifier from.</param>
|
/// <param name="entity">The entity to read the identifier from.</param>
|
||||||
/// <returns>The identifier value.</returns>
|
/// <returns>The identifier value.</returns>
|
||||||
public abstract TId GetId(T entity);
|
public abstract TId GetId(T entity);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sets the identifier value on an entity.
|
/// Sets the identifier value on an entity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity to update.</param>
|
/// <param name="entity">The entity to update.</param>
|
||||||
/// <param name="id">The identifier value to assign.</param>
|
/// <param name="id">The identifier value to assign.</param>
|
||||||
public abstract void SetId(T entity, TId id);
|
public abstract void SetId(T entity, TId id);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts a typed identifier value into an index key.
|
/// Converts a typed identifier value into an index key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="id">The identifier value.</param>
|
/// <param name="id">The identifier value.</param>
|
||||||
/// <returns>The index key representation of the identifier.</returns>
|
/// <returns>The index key representation of the identifier.</returns>
|
||||||
public virtual IndexKey ToIndexKey(TId id) => IndexKey.Create(id);
|
public virtual IndexKey ToIndexKey(TId id)
|
||||||
|
{
|
||||||
|
return IndexKey.Create(id);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts an index key back into a typed identifier value.
|
/// Converts an index key back into a typed identifier value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key to convert.</param>
|
/// <param name="key">The index key to convert.</param>
|
||||||
/// <returns>The typed identifier value.</returns>
|
/// <returns>The typed identifier value.</returns>
|
||||||
public virtual TId FromIndexKey(IndexKey key) => key.As<TId>();
|
public virtual TId FromIndexKey(IndexKey key)
|
||||||
|
{
|
||||||
|
return key.As<TId>();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets all mapped field keys used by this mapper.
|
/// Gets all mapped field keys used by this mapper.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public virtual IEnumerable<string> UsedKeys => GetSchema().GetAllKeys();
|
public virtual IEnumerable<string> UsedKeys => GetSchema().GetAllKeys();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Builds the BSON schema for the mapped entity type.
|
/// Builds the BSON schema for the mapped entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The generated BSON schema.</returns>
|
/// <returns>The generated BSON schema.</returns>
|
||||||
public virtual BsonSchema GetSchema() => BsonSchemaGenerator.FromType<T>();
|
public virtual BsonSchema GetSchema()
|
||||||
|
{
|
||||||
|
return BsonSchemaGenerator.FromType<T>();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Base class for mappers using ObjectId as primary key.
|
/// Base class for mappers using ObjectId as primary key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class ObjectIdMapperBase<T> : DocumentMapperBase<ObjectId, T>, IDocumentMapper<T> where T : class
|
public abstract class ObjectIdMapperBase<T> : DocumentMapperBase<ObjectId, T>, IDocumentMapper<T> where T : class
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override IndexKey ToIndexKey(ObjectId id) => IndexKey.Create(id);
|
public override IndexKey ToIndexKey(ObjectId id)
|
||||||
|
{
|
||||||
|
return IndexKey.Create(id);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override ObjectId FromIndexKey(IndexKey key) => key.As<ObjectId>();
|
public override ObjectId FromIndexKey(IndexKey key)
|
||||||
|
{
|
||||||
|
return key.As<ObjectId>();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Base class for mappers using Int32 as primary key.
|
/// Base class for mappers using Int32 as primary key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class Int32MapperBase<T> : DocumentMapperBase<int, T> where T : class
|
public abstract class Int32MapperBase<T> : DocumentMapperBase<int, T> where T : class
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override IndexKey ToIndexKey(int id) => IndexKey.Create(id);
|
public override IndexKey ToIndexKey(int id)
|
||||||
|
{
|
||||||
|
return IndexKey.Create(id);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override int FromIndexKey(IndexKey key) => key.As<int>();
|
public override int FromIndexKey(IndexKey key)
|
||||||
|
{
|
||||||
|
return key.As<int>();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Base class for mappers using String as primary key.
|
/// Base class for mappers using String as primary key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class StringMapperBase<T> : DocumentMapperBase<string, T> where T : class
|
public abstract class StringMapperBase<T> : DocumentMapperBase<string, T> where T : class
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override IndexKey ToIndexKey(string id) => IndexKey.Create(id);
|
public override IndexKey ToIndexKey(string id)
|
||||||
|
{
|
||||||
|
return IndexKey.Create(id);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string FromIndexKey(IndexKey key) => key.As<string>();
|
public override string FromIndexKey(IndexKey key)
|
||||||
|
{
|
||||||
|
return key.As<string>();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Base class for mappers using Guid as primary key.
|
/// Base class for mappers using Guid as primary key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class GuidMapperBase<T> : DocumentMapperBase<Guid, T> where T : class
|
public abstract class GuidMapperBase<T> : DocumentMapperBase<Guid, T> where T : class
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override IndexKey ToIndexKey(Guid id) => IndexKey.Create(id);
|
public override IndexKey ToIndexKey(Guid id)
|
||||||
|
{
|
||||||
|
return IndexKey.Create(id);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override Guid FromIndexKey(IndexKey key) => key.As<Guid>();
|
public override Guid FromIndexKey(IndexKey key)
|
||||||
}
|
{
|
||||||
|
return key.As<Guid>();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,36 +1,33 @@
|
|||||||
using System.Reflection;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
|
using System.Reflection;
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using System;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
|
|
||||||
public static class BsonSchemaGenerator
|
public static class BsonSchemaGenerator
|
||||||
{
|
{
|
||||||
/// <summary>
|
private static readonly ConcurrentDictionary<Type, BsonSchema> _cache = new();
|
||||||
/// Generates a BSON schema for the specified CLR type.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T">The CLR type to inspect.</typeparam>
|
|
||||||
/// <returns>The generated BSON schema.</returns>
|
|
||||||
public static BsonSchema FromType<T>()
|
|
||||||
{
|
|
||||||
return FromType(typeof(T));
|
|
||||||
}
|
|
||||||
|
|
||||||
private static readonly ConcurrentDictionary<Type, BsonSchema> _cache = new();
|
/// <summary>
|
||||||
|
/// Generates a BSON schema for the specified CLR type.
|
||||||
/// <summary>
|
/// </summary>
|
||||||
/// Generates a BSON schema for the specified CLR type.
|
/// <typeparam name="T">The CLR type to inspect.</typeparam>
|
||||||
/// </summary>
|
/// <returns>The generated BSON schema.</returns>
|
||||||
/// <param name="type">The CLR type to inspect.</param>
|
public static BsonSchema FromType<T>()
|
||||||
/// <returns>The generated BSON schema.</returns>
|
{
|
||||||
public static BsonSchema FromType(Type type)
|
return FromType(typeof(T));
|
||||||
{
|
}
|
||||||
return _cache.GetOrAdd(type, GenerateSchema);
|
|
||||||
|
/// <summary>
|
||||||
|
/// Generates a BSON schema for the specified CLR type.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="type">The CLR type to inspect.</param>
|
||||||
|
/// <returns>The generated BSON schema.</returns>
|
||||||
|
public static BsonSchema FromType(Type type)
|
||||||
|
{
|
||||||
|
return _cache.GetOrAdd(type, GenerateSchema);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static BsonSchema GenerateSchema(Type type)
|
private static BsonSchema GenerateSchema(Type type)
|
||||||
@@ -47,10 +44,7 @@ public static class BsonSchemaGenerator
|
|||||||
AddField(schema, prop.Name, prop.PropertyType);
|
AddField(schema, prop.Name, prop.PropertyType);
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var field in fields)
|
foreach (var field in fields) AddField(schema, field.Name, field.FieldType);
|
||||||
{
|
|
||||||
AddField(schema, field.Name, field.FieldType);
|
|
||||||
}
|
|
||||||
|
|
||||||
return schema;
|
return schema;
|
||||||
}
|
}
|
||||||
@@ -60,10 +54,7 @@ public static class BsonSchemaGenerator
|
|||||||
name = name.ToLowerInvariant();
|
name = name.ToLowerInvariant();
|
||||||
|
|
||||||
// Convention: id -> _id for root document
|
// Convention: id -> _id for root document
|
||||||
if (name.Equals("id", StringComparison.OrdinalIgnoreCase))
|
if (name.Equals("id", StringComparison.OrdinalIgnoreCase)) name = "_id";
|
||||||
{
|
|
||||||
name = "_id";
|
|
||||||
}
|
|
||||||
|
|
||||||
var (bsonType, nestedSchema, itemType) = GetBsonType(type);
|
var (bsonType, nestedSchema, itemType) = GetBsonType(type);
|
||||||
|
|
||||||
@@ -97,20 +88,18 @@ public static class BsonSchemaGenerator
|
|||||||
if (type != typeof(string) && typeof(IEnumerable).IsAssignableFrom(type))
|
if (type != typeof(string) && typeof(IEnumerable).IsAssignableFrom(type))
|
||||||
{
|
{
|
||||||
var itemType = GetCollectionItemType(type);
|
var itemType = GetCollectionItemType(type);
|
||||||
var (itemBsonType, itemNested, _) = GetBsonType(itemType);
|
var (itemBsonType, itemNested, _) = GetBsonType(itemType);
|
||||||
|
|
||||||
// For arrays, if item is Document, we use NestedSchema to describe the item
|
// For arrays, if item is Document, we use NestedSchema to describe the item
|
||||||
return (BsonType.Array, itemNested, itemBsonType);
|
return (BsonType.Array, itemNested, itemBsonType);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nested Objects / Structs
|
// Nested Objects / Structs
|
||||||
// If it's not a string, not a primitive, and not an array/list, treat as Document
|
// If it's not a string, not a primitive, and not an array/list, treat as Document
|
||||||
if (type != typeof(string) && !type.IsPrimitive && !type.IsEnum)
|
if (type != typeof(string) && !type.IsPrimitive && !type.IsEnum)
|
||||||
{
|
|
||||||
// Avoid infinite recursion?
|
// Avoid infinite recursion?
|
||||||
// Simple approach: generating nested schema
|
// Simple approach: generating nested schema
|
||||||
return (BsonType.Document, FromType(type), null);
|
return (BsonType.Document, FromType(type), null);
|
||||||
}
|
|
||||||
|
|
||||||
return (BsonType.Undefined, null, null);
|
return (BsonType.Undefined, null, null);
|
||||||
}
|
}
|
||||||
@@ -122,17 +111,15 @@ public static class BsonSchemaGenerator
|
|||||||
|
|
||||||
private static Type GetCollectionItemType(Type type)
|
private static Type GetCollectionItemType(Type type)
|
||||||
{
|
{
|
||||||
if (type.IsArray) return type.GetElementType()!;
|
if (type.IsArray) return type.GetElementType()!;
|
||||||
|
|
||||||
// If type itself is IEnumerable<T>
|
// If type itself is IEnumerable<T>
|
||||||
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>))
|
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IEnumerable<>))
|
||||||
{
|
|
||||||
return type.GetGenericArguments()[0];
|
return type.GetGenericArguments()[0];
|
||||||
}
|
|
||||||
|
|
||||||
var enumerableType = type.GetInterfaces()
|
var enumerableType = type.GetInterfaces()
|
||||||
.FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>));
|
.FirstOrDefault(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEnumerable<>));
|
||||||
|
|
||||||
return enumerableType?.GetGenericArguments()[0] ?? typeof(object);
|
return enumerableType?.GetGenericArguments()[0] ?? typeof(object);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -8,8 +8,8 @@ namespace ZB.MOM.WW.CBDD.Core.Collections;
|
|||||||
public partial class DocumentCollection<TId, T> where T : class
|
public partial class DocumentCollection<TId, T> where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Scans the entire collection using a raw BSON predicate.
|
/// Scans the entire collection using a raw BSON predicate.
|
||||||
/// This avoids deserializing documents that don't match the criteria.
|
/// This avoids deserializing documents that don't match the criteria.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
||||||
/// <returns>Matching documents</returns>
|
/// <returns>Matching documents</returns>
|
||||||
@@ -18,8 +18,8 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
||||||
|
|
||||||
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
||||||
var txnId = transaction.TransactionId;
|
ulong txnId = transaction.TransactionId;
|
||||||
var pageCount = _storage.PageCount;
|
uint pageCount = _storage.PageCount;
|
||||||
var buffer = new byte[_storage.PageSize];
|
var buffer = new byte[_storage.PageSize];
|
||||||
var pageResults = new List<T>();
|
var pageResults = new List<T>();
|
||||||
|
|
||||||
@@ -28,16 +28,13 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
pageResults.Clear();
|
pageResults.Clear();
|
||||||
ScanPage(pageId, txnId, buffer, predicate, pageResults);
|
ScanPage(pageId, txnId, buffer, predicate, pageResults);
|
||||||
|
|
||||||
foreach (var doc in pageResults)
|
foreach (var doc in pageResults) yield return doc;
|
||||||
{
|
|
||||||
yield return doc;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Scans the collection in parallel using multiple threads.
|
/// Scans the collection in parallel using multiple threads.
|
||||||
/// Useful for large collections on multi-core machines.
|
/// Useful for large collections on multi-core machines.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
/// <param name="predicate">Function to evaluate raw BSON data</param>
|
||||||
/// <param name="degreeOfParallelism">Number of threads to use (default: -1 = ProcessorCount)</param>
|
/// <param name="degreeOfParallelism">Number of threads to use (default: -1 = ProcessorCount)</param>
|
||||||
@@ -46,7 +43,7 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
if (predicate == null) throw new ArgumentNullException(nameof(predicate));
|
||||||
|
|
||||||
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
var transaction = _transactionHolder.GetCurrentTransactionOrStart();
|
||||||
var txnId = transaction.TransactionId;
|
ulong txnId = transaction.TransactionId;
|
||||||
var pageCount = (int)_storage.PageCount;
|
var pageCount = (int)_storage.PageCount;
|
||||||
|
|
||||||
if (degreeOfParallelism <= 0)
|
if (degreeOfParallelism <= 0)
|
||||||
@@ -61,15 +58,14 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
var localResults = new List<T>();
|
var localResults = new List<T>();
|
||||||
|
|
||||||
for (int i = range.Item1; i < range.Item2; i++)
|
for (int i = range.Item1; i < range.Item2; i++)
|
||||||
{
|
|
||||||
ScanPage((uint)i, txnId, localBuffer, predicate, localResults);
|
ScanPage((uint)i, txnId, localBuffer, predicate, localResults);
|
||||||
}
|
|
||||||
|
|
||||||
return localResults;
|
return localResults;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private void ScanPage(uint pageId, ulong txnId, byte[] buffer, Func<BsonSpanReader, bool> predicate, List<T> results)
|
private void ScanPage(uint pageId, ulong txnId, byte[] buffer, Func<BsonSpanReader, bool> predicate,
|
||||||
|
List<T> results)
|
||||||
{
|
{
|
||||||
_storage.ReadPage(pageId, txnId, buffer);
|
_storage.ReadPage(pageId, txnId, buffer);
|
||||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||||
@@ -80,7 +76,7 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
var slots = MemoryMarshal.Cast<byte, SlotEntry>(
|
var slots = MemoryMarshal.Cast<byte, SlotEntry>(
|
||||||
buffer.AsSpan(SlottedPageHeader.Size, header.SlotCount * SlotEntry.Size));
|
buffer.AsSpan(SlottedPageHeader.Size, header.SlotCount * SlotEntry.Size));
|
||||||
|
|
||||||
for (int i = 0; i < header.SlotCount; i++)
|
for (var i = 0; i < header.SlotCount; i++)
|
||||||
{
|
{
|
||||||
var slot = slots[i];
|
var slot = slots[i];
|
||||||
|
|
||||||
@@ -98,4 +94,4 @@ public partial class DocumentCollection<TId, T> where T : class
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,42 +1,39 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
using System;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
using System.Buffers;
|
|
||||||
using System.Collections.Generic;
|
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
|
||||||
|
/// <summary>
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
/// Non-generic interface for common mapper operations.
|
||||||
|
/// </summary>
|
||||||
/// <summary>
|
|
||||||
/// Non-generic interface for common mapper operations.
|
|
||||||
/// </summary>
|
|
||||||
public interface IDocumentMapper
|
public interface IDocumentMapper
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection name handled by this mapper.
|
/// Gets the collection name handled by this mapper.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
string CollectionName { get; }
|
string CollectionName { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the set of document keys used during mapping.
|
/// Gets the set of document keys used during mapping.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
IEnumerable<string> UsedKeys { get; }
|
IEnumerable<string> UsedKeys { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the BSON schema for the mapped document.
|
/// Gets the BSON schema for the mapped document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The BSON schema.</returns>
|
/// <returns>The BSON schema.</returns>
|
||||||
BsonSchema GetSchema();
|
BsonSchema GetSchema();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Interface for mapping between entities and BSON using zero-allocation serialization.
|
/// Interface for mapping between entities and BSON using zero-allocation serialization.
|
||||||
/// Handles bidirectional mapping between TId and IndexKey.
|
/// Handles bidirectional mapping between TId and IndexKey.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public interface IDocumentMapper<TId, T> : IDocumentMapper where T : class
|
public interface IDocumentMapper<TId, T> : IDocumentMapper where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Serializes an entity to BSON.
|
/// Serializes an entity to BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity to serialize.</param>
|
/// <param name="entity">The entity to serialize.</param>
|
||||||
/// <param name="writer">The BSON writer.</param>
|
/// <param name="writer">The BSON writer.</param>
|
||||||
@@ -44,44 +41,44 @@ public interface IDocumentMapper<TId, T> : IDocumentMapper where T : class
|
|||||||
int Serialize(T entity, BsonSpanWriter writer);
|
int Serialize(T entity, BsonSpanWriter writer);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deserializes an entity from BSON.
|
/// Deserializes an entity from BSON.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="reader">The BSON reader.</param>
|
/// <param name="reader">The BSON reader.</param>
|
||||||
/// <returns>The deserialized entity.</returns>
|
/// <returns>The deserialized entity.</returns>
|
||||||
T Deserialize(BsonSpanReader reader);
|
T Deserialize(BsonSpanReader reader);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the identifier value from an entity.
|
/// Gets the identifier value from an entity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity.</param>
|
/// <param name="entity">The entity.</param>
|
||||||
/// <returns>The identifier value.</returns>
|
/// <returns>The identifier value.</returns>
|
||||||
TId GetId(T entity);
|
TId GetId(T entity);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sets the identifier value on an entity.
|
/// Sets the identifier value on an entity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="entity">The entity.</param>
|
/// <param name="entity">The entity.</param>
|
||||||
/// <param name="id">The identifier value.</param>
|
/// <param name="id">The identifier value.</param>
|
||||||
void SetId(T entity, TId id);
|
void SetId(T entity, TId id);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts an identifier to an index key.
|
/// Converts an identifier to an index key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="id">The identifier value.</param>
|
/// <param name="id">The identifier value.</param>
|
||||||
/// <returns>The index key representation.</returns>
|
/// <returns>The index key representation.</returns>
|
||||||
IndexKey ToIndexKey(TId id);
|
IndexKey ToIndexKey(TId id);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts an index key back to an identifier.
|
/// Converts an index key back to an identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <returns>The identifier value.</returns>
|
/// <returns>The identifier value.</returns>
|
||||||
TId FromIndexKey(IndexKey key);
|
TId FromIndexKey(IndexKey key);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Legacy interface for compatibility with existing ObjectId-based collections.
|
/// Legacy interface for compatibility with existing ObjectId-based collections.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public interface IDocumentMapper<T> : IDocumentMapper<ObjectId, T> where T : class
|
public interface IDocumentMapper<T> : IDocumentMapper<ObjectId, T> where T : class
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
@@ -1,21 +1,19 @@
|
|||||||
using System;
|
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Collections;
|
|
||||||
|
|
||||||
public readonly struct SchemaVersion
|
public readonly struct SchemaVersion
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the schema version number.
|
/// Gets the schema version number.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Version { get; }
|
public int Version { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the schema hash.
|
/// Gets the schema hash.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long Hash { get; }
|
public long Hash { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="SchemaVersion"/> struct.
|
/// Initializes a new instance of the <see cref="SchemaVersion" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="version">The schema version number.</param>
|
/// <param name="version">The schema version number.</param>
|
||||||
/// <param name="hash">The schema hash.</param>
|
/// <param name="hash">The schema hash.</param>
|
||||||
@@ -26,5 +24,8 @@ public readonly struct SchemaVersion
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string ToString() => $"v{Version} (0x{Hash:X16})";
|
public override string ToString()
|
||||||
}
|
{
|
||||||
|
return $"v{Version} (0x{Hash:X16})";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,34 +3,34 @@ using System.Buffers.Binary;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Fixed header prefix for compressed payload blobs.
|
/// Fixed header prefix for compressed payload blobs.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly struct CompressedPayloadHeader
|
public readonly struct CompressedPayloadHeader
|
||||||
{
|
{
|
||||||
public const int Size = 16;
|
public const int Size = 16;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compression codec used for payload bytes.
|
/// Compression codec used for payload bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionCodec Codec { get; }
|
public CompressionCodec Codec { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Original uncompressed payload length.
|
/// Original uncompressed payload length.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int OriginalLength { get; }
|
public int OriginalLength { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compressed payload length.
|
/// Compressed payload length.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int CompressedLength { get; }
|
public int CompressedLength { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// CRC32 checksum of compressed payload bytes.
|
/// CRC32 checksum of compressed payload bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint Checksum { get; }
|
public uint Checksum { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="CompressedPayloadHeader"/> class.
|
/// Initializes a new instance of the <see cref="CompressedPayloadHeader" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="codec">Compression codec used for payload bytes.</param>
|
/// <param name="codec">Compression codec used for payload bytes.</param>
|
||||||
/// <param name="originalLength">Original uncompressed payload length.</param>
|
/// <param name="originalLength">Original uncompressed payload length.</param>
|
||||||
@@ -50,19 +50,20 @@ public readonly struct CompressedPayloadHeader
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Create.
|
/// Create.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="codec">Compression codec used for payload bytes.</param>
|
/// <param name="codec">Compression codec used for payload bytes.</param>
|
||||||
/// <param name="originalLength">Original uncompressed payload length.</param>
|
/// <param name="originalLength">Original uncompressed payload length.</param>
|
||||||
/// <param name="compressedPayload">Compressed payload bytes.</param>
|
/// <param name="compressedPayload">Compressed payload bytes.</param>
|
||||||
public static CompressedPayloadHeader Create(CompressionCodec codec, int originalLength, ReadOnlySpan<byte> compressedPayload)
|
public static CompressedPayloadHeader Create(CompressionCodec codec, int originalLength,
|
||||||
|
ReadOnlySpan<byte> compressedPayload)
|
||||||
{
|
{
|
||||||
var checksum = ComputeChecksum(compressedPayload);
|
uint checksum = ComputeChecksum(compressedPayload);
|
||||||
return new CompressedPayloadHeader(codec, originalLength, compressedPayload.Length, checksum);
|
return new CompressedPayloadHeader(codec, originalLength, compressedPayload.Length, checksum);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Write To.
|
/// Write To.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">Destination span that receives the serialized header.</param>
|
/// <param name="destination">Destination span that receives the serialized header.</param>
|
||||||
public void WriteTo(Span<byte> destination)
|
public void WriteTo(Span<byte> destination)
|
||||||
@@ -80,7 +81,7 @@ public readonly struct CompressedPayloadHeader
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Read From.
|
/// Read From.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">Source span containing a serialized header.</param>
|
/// <param name="source">Source span containing a serialized header.</param>
|
||||||
public static CompressedPayloadHeader ReadFrom(ReadOnlySpan<byte> source)
|
public static CompressedPayloadHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||||
@@ -89,14 +90,14 @@ public readonly struct CompressedPayloadHeader
|
|||||||
throw new ArgumentException($"Source must be at least {Size} bytes.", nameof(source));
|
throw new ArgumentException($"Source must be at least {Size} bytes.", nameof(source));
|
||||||
|
|
||||||
var codec = (CompressionCodec)source[0];
|
var codec = (CompressionCodec)source[0];
|
||||||
var originalLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(4, 4));
|
int originalLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(4, 4));
|
||||||
var compressedLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(8, 4));
|
int compressedLength = BinaryPrimitives.ReadInt32LittleEndian(source.Slice(8, 4));
|
||||||
var checksum = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(12, 4));
|
uint checksum = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(12, 4));
|
||||||
return new CompressedPayloadHeader(codec, originalLength, compressedLength, checksum);
|
return new CompressedPayloadHeader(codec, originalLength, compressedLength, checksum);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Validate Checksum.
|
/// Validate Checksum.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="compressedPayload">Compressed payload bytes to validate.</param>
|
/// <param name="compressedPayload">Compressed payload bytes to validate.</param>
|
||||||
public bool ValidateChecksum(ReadOnlySpan<byte> compressedPayload)
|
public bool ValidateChecksum(ReadOnlySpan<byte> compressedPayload)
|
||||||
@@ -105,10 +106,13 @@ public readonly struct CompressedPayloadHeader
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compute Checksum.
|
/// Compute Checksum.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="payload">Payload bytes.</param>
|
/// <param name="payload">Payload bytes.</param>
|
||||||
public static uint ComputeChecksum(ReadOnlySpan<byte> payload) => Crc32Calculator.Compute(payload);
|
public static uint ComputeChecksum(ReadOnlySpan<byte> payload)
|
||||||
|
{
|
||||||
|
return Crc32Calculator.Compute(payload);
|
||||||
|
}
|
||||||
|
|
||||||
private static class Crc32Calculator
|
private static class Crc32Calculator
|
||||||
{
|
{
|
||||||
@@ -116,15 +120,15 @@ public readonly struct CompressedPayloadHeader
|
|||||||
private static readonly uint[] Table = CreateTable();
|
private static readonly uint[] Table = CreateTable();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compute.
|
/// Compute.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="payload">Payload bytes.</param>
|
/// <param name="payload">Payload bytes.</param>
|
||||||
public static uint Compute(ReadOnlySpan<byte> payload)
|
public static uint Compute(ReadOnlySpan<byte> payload)
|
||||||
{
|
{
|
||||||
uint crc = 0xFFFFFFFFu;
|
var crc = 0xFFFFFFFFu;
|
||||||
for (int i = 0; i < payload.Length; i++)
|
for (var i = 0; i < payload.Length; i++)
|
||||||
{
|
{
|
||||||
var index = (crc ^ payload[i]) & 0xFF;
|
uint index = (crc ^ payload[i]) & 0xFF;
|
||||||
crc = (crc >> 8) ^ Table[index];
|
crc = (crc >> 8) ^ Table[index];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,10 +141,7 @@ public readonly struct CompressedPayloadHeader
|
|||||||
for (uint i = 0; i < table.Length; i++)
|
for (uint i = 0; i < table.Length; i++)
|
||||||
{
|
{
|
||||||
uint value = i;
|
uint value = i;
|
||||||
for (int bit = 0; bit < 8; bit++)
|
for (var bit = 0; bit < 8; bit++) value = (value & 1) != 0 ? (value >> 1) ^ Polynomial : value >> 1;
|
||||||
{
|
|
||||||
value = (value & 1) != 0 ? (value >> 1) ^ Polynomial : value >> 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
table[i] = value;
|
table[i] = value;
|
||||||
}
|
}
|
||||||
@@ -148,4 +149,4 @@ public readonly struct CompressedPayloadHeader
|
|||||||
return table;
|
return table;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Supported payload compression codecs.
|
/// Supported payload compression codecs.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum CompressionCodec : byte
|
public enum CompressionCodec : byte
|
||||||
{
|
{
|
||||||
None = 0,
|
None = 0,
|
||||||
Brotli = 1,
|
Brotli = 1,
|
||||||
Deflate = 2
|
Deflate = 2
|
||||||
}
|
}
|
||||||
@@ -3,52 +3,52 @@ using System.IO.Compression;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compression configuration for document payload processing.
|
/// Compression configuration for document payload processing.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompressionOptions
|
public sealed class CompressionOptions
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Default compression options (compression disabled).
|
/// Default compression options (compression disabled).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static CompressionOptions Default { get; } = new();
|
public static CompressionOptions Default { get; } = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Enables payload compression for new writes.
|
/// Enables payload compression for new writes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool EnableCompression { get; init; } = false;
|
public bool EnableCompression { get; init; } = false;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Minimum payload size (bytes) required before compression is attempted.
|
/// Minimum payload size (bytes) required before compression is attempted.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int MinSizeBytes { get; init; } = 1024;
|
public int MinSizeBytes { get; init; } = 1024;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Minimum percentage of size reduction required to keep compressed output.
|
/// Minimum percentage of size reduction required to keep compressed output.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int MinSavingsPercent { get; init; } = 10;
|
public int MinSavingsPercent { get; init; } = 10;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Preferred default codec for new writes.
|
/// Preferred default codec for new writes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compression level passed to codec implementations.
|
/// Compression level passed to codec implementations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Maximum allowed decompressed payload size.
|
/// Maximum allowed decompressed payload size.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int MaxDecompressedSizeBytes { get; init; } = 16 * 1024 * 1024;
|
public int MaxDecompressedSizeBytes { get; init; } = 16 * 1024 * 1024;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Optional maximum input size allowed for compression attempts.
|
/// Optional maximum input size allowed for compression attempts.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int? MaxCompressionInputBytes { get; init; }
|
public int? MaxCompressionInputBytes { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Normalizes and validates compression options.
|
/// Normalizes and validates compression options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Optional user-provided options.</param>
|
/// <param name="options">Optional user-provided options.</param>
|
||||||
internal static CompressionOptions Normalize(CompressionOptions? options)
|
internal static CompressionOptions Normalize(CompressionOptions? options)
|
||||||
@@ -59,17 +59,20 @@ public sealed class CompressionOptions
|
|||||||
throw new ArgumentOutOfRangeException(nameof(MinSizeBytes), "MinSizeBytes must be non-negative.");
|
throw new ArgumentOutOfRangeException(nameof(MinSizeBytes), "MinSizeBytes must be non-negative.");
|
||||||
|
|
||||||
if (candidate.MinSavingsPercent is < 0 or > 100)
|
if (candidate.MinSavingsPercent is < 0 or > 100)
|
||||||
throw new ArgumentOutOfRangeException(nameof(MinSavingsPercent), "MinSavingsPercent must be between 0 and 100.");
|
throw new ArgumentOutOfRangeException(nameof(MinSavingsPercent),
|
||||||
|
"MinSavingsPercent must be between 0 and 100.");
|
||||||
|
|
||||||
if (!Enum.IsDefined(candidate.Codec))
|
if (!Enum.IsDefined(candidate.Codec))
|
||||||
throw new ArgumentOutOfRangeException(nameof(Codec), $"Unsupported codec: {candidate.Codec}.");
|
throw new ArgumentOutOfRangeException(nameof(Codec), $"Unsupported codec: {candidate.Codec}.");
|
||||||
|
|
||||||
if (candidate.MaxDecompressedSizeBytes <= 0)
|
if (candidate.MaxDecompressedSizeBytes <= 0)
|
||||||
throw new ArgumentOutOfRangeException(nameof(MaxDecompressedSizeBytes), "MaxDecompressedSizeBytes must be greater than 0.");
|
throw new ArgumentOutOfRangeException(nameof(MaxDecompressedSizeBytes),
|
||||||
|
"MaxDecompressedSizeBytes must be greater than 0.");
|
||||||
|
|
||||||
if (candidate.MaxCompressionInputBytes is <= 0)
|
if (candidate.MaxCompressionInputBytes is <= 0)
|
||||||
throw new ArgumentOutOfRangeException(nameof(MaxCompressionInputBytes), "MaxCompressionInputBytes must be greater than 0 when provided.");
|
throw new ArgumentOutOfRangeException(nameof(MaxCompressionInputBytes),
|
||||||
|
"MaxCompressionInputBytes must be greater than 0 when provided.");
|
||||||
|
|
||||||
return candidate;
|
return candidate;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,14 +5,14 @@ using System.IO.Compression;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compression codec registry and utility service.
|
/// Compression codec registry and utility service.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompressionService
|
public sealed class CompressionService
|
||||||
{
|
{
|
||||||
private readonly ConcurrentDictionary<CompressionCodec, ICompressionCodec> _codecs = new();
|
private readonly ConcurrentDictionary<CompressionCodec, ICompressionCodec> _codecs = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="CompressionService"/> class.
|
/// Initializes a new instance of the <see cref="CompressionService" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="additionalCodecs">Optional additional codecs to register.</param>
|
/// <param name="additionalCodecs">Optional additional codecs to register.</param>
|
||||||
public CompressionService(IEnumerable<ICompressionCodec>? additionalCodecs = null)
|
public CompressionService(IEnumerable<ICompressionCodec>? additionalCodecs = null)
|
||||||
@@ -24,14 +24,11 @@ public sealed class CompressionService
|
|||||||
if (additionalCodecs == null)
|
if (additionalCodecs == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
foreach (var codec in additionalCodecs)
|
foreach (var codec in additionalCodecs) RegisterCodec(codec);
|
||||||
{
|
|
||||||
RegisterCodec(codec);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Registers or replaces a compression codec implementation.
|
/// Registers or replaces a compression codec implementation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="codec">The codec implementation to register.</param>
|
/// <param name="codec">The codec implementation to register.</param>
|
||||||
public void RegisterCodec(ICompressionCodec codec)
|
public void RegisterCodec(ICompressionCodec codec)
|
||||||
@@ -41,18 +38,21 @@ public sealed class CompressionService
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Attempts to resolve a registered codec implementation.
|
/// Attempts to resolve a registered codec implementation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="codec">The codec identifier to resolve.</param>
|
/// <param name="codec">The codec identifier to resolve.</param>
|
||||||
/// <param name="compressionCodec">When this method returns, contains the resolved codec when found.</param>
|
/// <param name="compressionCodec">When this method returns, contains the resolved codec when found.</param>
|
||||||
/// <returns><see langword="true"/> when a codec is registered for <paramref name="codec"/>; otherwise, <see langword="false"/>.</returns>
|
/// <returns>
|
||||||
|
/// <see langword="true" /> when a codec is registered for <paramref name="codec" />; otherwise,
|
||||||
|
/// <see langword="false" />.
|
||||||
|
/// </returns>
|
||||||
public bool TryGetCodec(CompressionCodec codec, out ICompressionCodec compressionCodec)
|
public bool TryGetCodec(CompressionCodec codec, out ICompressionCodec compressionCodec)
|
||||||
{
|
{
|
||||||
return _codecs.TryGetValue(codec, out compressionCodec!);
|
return _codecs.TryGetValue(codec, out compressionCodec!);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a registered codec implementation.
|
/// Gets a registered codec implementation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="codec">The codec identifier to resolve.</param>
|
/// <param name="codec">The codec identifier to resolve.</param>
|
||||||
/// <returns>The registered codec implementation.</returns>
|
/// <returns>The registered codec implementation.</returns>
|
||||||
@@ -65,7 +65,7 @@ public sealed class CompressionService
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compresses payload bytes using the selected codec and level.
|
/// Compresses payload bytes using the selected codec and level.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="input">The payload bytes to compress.</param>
|
/// <param name="input">The payload bytes to compress.</param>
|
||||||
/// <param name="codec">The codec to use.</param>
|
/// <param name="codec">The codec to use.</param>
|
||||||
@@ -77,131 +77,40 @@ public sealed class CompressionService
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Decompresses payload bytes using the selected codec.
|
/// Decompresses payload bytes using the selected codec.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="input">The compressed payload bytes.</param>
|
/// <param name="input">The compressed payload bytes.</param>
|
||||||
/// <param name="codec">The codec to use.</param>
|
/// <param name="codec">The codec to use.</param>
|
||||||
/// <param name="expectedLength">The expected decompressed byte length, or a negative value to skip exact-length validation.</param>
|
/// <param name="expectedLength">
|
||||||
|
/// The expected decompressed byte length, or a negative value to skip exact-length
|
||||||
|
/// validation.
|
||||||
|
/// </param>
|
||||||
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
||||||
/// <returns>The decompressed payload bytes.</returns>
|
/// <returns>The decompressed payload bytes.</returns>
|
||||||
public byte[] Decompress(ReadOnlySpan<byte> input, CompressionCodec codec, int expectedLength, int maxDecompressedSizeBytes)
|
public byte[] Decompress(ReadOnlySpan<byte> input, CompressionCodec codec, int expectedLength,
|
||||||
|
int maxDecompressedSizeBytes)
|
||||||
{
|
{
|
||||||
return GetCodec(codec).Decompress(input, expectedLength, maxDecompressedSizeBytes);
|
return GetCodec(codec).Decompress(input, expectedLength, maxDecompressedSizeBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compresses and then decompresses payload bytes using the selected codec.
|
/// Compresses and then decompresses payload bytes using the selected codec.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="input">The payload bytes to roundtrip.</param>
|
/// <param name="input">The payload bytes to roundtrip.</param>
|
||||||
/// <param name="codec">The codec to use.</param>
|
/// <param name="codec">The codec to use.</param>
|
||||||
/// <param name="level">The compression level.</param>
|
/// <param name="level">The compression level.</param>
|
||||||
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
||||||
/// <returns>The decompressed payload bytes after roundtrip.</returns>
|
/// <returns>The decompressed payload bytes after roundtrip.</returns>
|
||||||
public byte[] Roundtrip(ReadOnlySpan<byte> input, CompressionCodec codec, CompressionLevel level, int maxDecompressedSizeBytes)
|
public byte[] Roundtrip(ReadOnlySpan<byte> input, CompressionCodec codec, CompressionLevel level,
|
||||||
|
int maxDecompressedSizeBytes)
|
||||||
{
|
{
|
||||||
var compressed = Compress(input, codec, level);
|
byte[] compressed = Compress(input, codec, level);
|
||||||
return Decompress(compressed, codec, input.Length, maxDecompressedSizeBytes);
|
return Decompress(compressed, codec, input.Length, maxDecompressedSizeBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private sealed class NoneCompressionCodec : ICompressionCodec
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the codec identifier.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionCodec Codec => CompressionCodec.None;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Returns a copy of the input payload without compression.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The payload bytes to copy.</param>
|
|
||||||
/// <param name="level">The requested compression level.</param>
|
|
||||||
/// <returns>The copied payload bytes.</returns>
|
|
||||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level) => input.ToArray();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Validates and returns an uncompressed payload copy.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The payload bytes to validate and copy.</param>
|
|
||||||
/// <param name="expectedLength">The expected payload length, or a negative value to skip exact-length validation.</param>
|
|
||||||
/// <param name="maxDecompressedSizeBytes">The maximum allowed payload size in bytes.</param>
|
|
||||||
/// <returns>The copied payload bytes.</returns>
|
|
||||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
|
||||||
{
|
|
||||||
if (input.Length > maxDecompressedSizeBytes)
|
|
||||||
throw new InvalidDataException($"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
|
||||||
|
|
||||||
if (expectedLength >= 0 && expectedLength != input.Length)
|
|
||||||
throw new InvalidDataException($"Expected decompressed length {expectedLength}, actual {input.Length}.");
|
|
||||||
|
|
||||||
return input.ToArray();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private sealed class BrotliCompressionCodec : ICompressionCodec
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the codec identifier.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionCodec Codec => CompressionCodec.Brotli;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compresses payload bytes using Brotli.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The payload bytes to compress.</param>
|
|
||||||
/// <param name="level">The compression level.</param>
|
|
||||||
/// <returns>The compressed payload bytes.</returns>
|
|
||||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
|
||||||
{
|
|
||||||
return CompressWithCodecStream(input, stream => new BrotliStream(stream, level, leaveOpen: true));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Decompresses Brotli-compressed payload bytes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The compressed payload bytes.</param>
|
|
||||||
/// <param name="expectedLength">The expected decompressed byte length, or a negative value to skip exact-length validation.</param>
|
|
||||||
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
|
||||||
/// <returns>The decompressed payload bytes.</returns>
|
|
||||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
|
||||||
{
|
|
||||||
return DecompressWithCodecStream(input, stream => new BrotliStream(stream, CompressionMode.Decompress, leaveOpen: true), expectedLength, maxDecompressedSizeBytes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private sealed class DeflateCompressionCodec : ICompressionCodec
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the codec identifier.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionCodec Codec => CompressionCodec.Deflate;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compresses payload bytes using Deflate.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The payload bytes to compress.</param>
|
|
||||||
/// <param name="level">The compression level.</param>
|
|
||||||
/// <returns>The compressed payload bytes.</returns>
|
|
||||||
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
|
||||||
{
|
|
||||||
return CompressWithCodecStream(input, stream => new DeflateStream(stream, level, leaveOpen: true));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Decompresses Deflate-compressed payload bytes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">The compressed payload bytes.</param>
|
|
||||||
/// <param name="expectedLength">The expected decompressed byte length, or a negative value to skip exact-length validation.</param>
|
|
||||||
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
|
||||||
/// <returns>The decompressed payload bytes.</returns>
|
|
||||||
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
|
||||||
{
|
|
||||||
return DecompressWithCodecStream(input, stream => new DeflateStream(stream, CompressionMode.Decompress, leaveOpen: true), expectedLength, maxDecompressedSizeBytes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static byte[] CompressWithCodecStream(ReadOnlySpan<byte> input, Func<Stream, Stream> streamFactory)
|
private static byte[] CompressWithCodecStream(ReadOnlySpan<byte> input, Func<Stream, Stream> streamFactory)
|
||||||
{
|
{
|
||||||
using var output = new MemoryStream(capacity: input.Length);
|
using var output = new MemoryStream(input.Length);
|
||||||
using (var codecStream = streamFactory(output))
|
using (var codecStream = streamFactory(output))
|
||||||
{
|
{
|
||||||
codecStream.Write(input);
|
codecStream.Write(input);
|
||||||
@@ -220,31 +129,33 @@ public sealed class CompressionService
|
|||||||
if (maxDecompressedSizeBytes <= 0)
|
if (maxDecompressedSizeBytes <= 0)
|
||||||
throw new ArgumentOutOfRangeException(nameof(maxDecompressedSizeBytes));
|
throw new ArgumentOutOfRangeException(nameof(maxDecompressedSizeBytes));
|
||||||
|
|
||||||
using var compressed = new MemoryStream(input.ToArray(), writable: false);
|
using var compressed = new MemoryStream(input.ToArray(), false);
|
||||||
using var codecStream = streamFactory(compressed);
|
using var codecStream = streamFactory(compressed);
|
||||||
using var output = expectedLength > 0
|
using var output = expectedLength > 0
|
||||||
? new MemoryStream(capacity: expectedLength)
|
? new MemoryStream(expectedLength)
|
||||||
: new MemoryStream();
|
: new MemoryStream();
|
||||||
|
|
||||||
var buffer = ArrayPool<byte>.Shared.Rent(8192);
|
byte[] buffer = ArrayPool<byte>.Shared.Rent(8192);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
int totalWritten = 0;
|
var totalWritten = 0;
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
var bytesRead = codecStream.Read(buffer, 0, buffer.Length);
|
int bytesRead = codecStream.Read(buffer, 0, buffer.Length);
|
||||||
if (bytesRead <= 0)
|
if (bytesRead <= 0)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
totalWritten += bytesRead;
|
totalWritten += bytesRead;
|
||||||
if (totalWritten > maxDecompressedSizeBytes)
|
if (totalWritten > maxDecompressedSizeBytes)
|
||||||
throw new InvalidDataException($"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
throw new InvalidDataException(
|
||||||
|
$"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
||||||
|
|
||||||
output.Write(buffer, 0, bytesRead);
|
output.Write(buffer, 0, bytesRead);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (expectedLength >= 0 && totalWritten != expectedLength)
|
if (expectedLength >= 0 && totalWritten != expectedLength)
|
||||||
throw new InvalidDataException($"Expected decompressed length {expectedLength}, actual {totalWritten}.");
|
throw new InvalidDataException(
|
||||||
|
$"Expected decompressed length {expectedLength}, actual {totalWritten}.");
|
||||||
|
|
||||||
return output.ToArray();
|
return output.ToArray();
|
||||||
}
|
}
|
||||||
@@ -253,4 +164,115 @@ public sealed class CompressionService
|
|||||||
ArrayPool<byte>.Shared.Return(buffer);
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
private sealed class NoneCompressionCodec : ICompressionCodec
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the codec identifier.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionCodec Codec => CompressionCodec.None;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns a copy of the input payload without compression.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The payload bytes to copy.</param>
|
||||||
|
/// <param name="level">The requested compression level.</param>
|
||||||
|
/// <returns>The copied payload bytes.</returns>
|
||||||
|
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
||||||
|
{
|
||||||
|
return input.ToArray();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Validates and returns an uncompressed payload copy.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The payload bytes to validate and copy.</param>
|
||||||
|
/// <param name="expectedLength">The expected payload length, or a negative value to skip exact-length validation.</param>
|
||||||
|
/// <param name="maxDecompressedSizeBytes">The maximum allowed payload size in bytes.</param>
|
||||||
|
/// <returns>The copied payload bytes.</returns>
|
||||||
|
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||||
|
{
|
||||||
|
if (input.Length > maxDecompressedSizeBytes)
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"Decompressed payload exceeds max allowed size ({maxDecompressedSizeBytes} bytes).");
|
||||||
|
|
||||||
|
if (expectedLength >= 0 && expectedLength != input.Length)
|
||||||
|
throw new InvalidDataException(
|
||||||
|
$"Expected decompressed length {expectedLength}, actual {input.Length}.");
|
||||||
|
|
||||||
|
return input.ToArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class BrotliCompressionCodec : ICompressionCodec
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the codec identifier.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionCodec Codec => CompressionCodec.Brotli;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compresses payload bytes using Brotli.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The payload bytes to compress.</param>
|
||||||
|
/// <param name="level">The compression level.</param>
|
||||||
|
/// <returns>The compressed payload bytes.</returns>
|
||||||
|
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
||||||
|
{
|
||||||
|
return CompressWithCodecStream(input, stream => new BrotliStream(stream, level, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Decompresses Brotli-compressed payload bytes.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The compressed payload bytes.</param>
|
||||||
|
/// <param name="expectedLength">
|
||||||
|
/// The expected decompressed byte length, or a negative value to skip exact-length
|
||||||
|
/// validation.
|
||||||
|
/// </param>
|
||||||
|
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
||||||
|
/// <returns>The decompressed payload bytes.</returns>
|
||||||
|
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||||
|
{
|
||||||
|
return DecompressWithCodecStream(input,
|
||||||
|
stream => new BrotliStream(stream, CompressionMode.Decompress, true), expectedLength,
|
||||||
|
maxDecompressedSizeBytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private sealed class DeflateCompressionCodec : ICompressionCodec
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the codec identifier.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionCodec Codec => CompressionCodec.Deflate;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compresses payload bytes using Deflate.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The payload bytes to compress.</param>
|
||||||
|
/// <param name="level">The compression level.</param>
|
||||||
|
/// <returns>The compressed payload bytes.</returns>
|
||||||
|
public byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level)
|
||||||
|
{
|
||||||
|
return CompressWithCodecStream(input, stream => new DeflateStream(stream, level, true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Decompresses Deflate-compressed payload bytes.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="input">The compressed payload bytes.</param>
|
||||||
|
/// <param name="expectedLength">
|
||||||
|
/// The expected decompressed byte length, or a negative value to skip exact-length
|
||||||
|
/// validation.
|
||||||
|
/// </param>
|
||||||
|
/// <param name="maxDecompressedSizeBytes">The maximum allowed decompressed byte length.</param>
|
||||||
|
/// <returns>The decompressed payload bytes.</returns>
|
||||||
|
public byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes)
|
||||||
|
{
|
||||||
|
return DecompressWithCodecStream(input,
|
||||||
|
stream => new DeflateStream(stream, CompressionMode.Decompress, true), expectedLength,
|
||||||
|
maxDecompressedSizeBytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,40 +1,47 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Snapshot of aggregated compression and decompression telemetry.
|
/// Snapshot of aggregated compression and decompression telemetry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly struct CompressionStats
|
public readonly struct CompressionStats
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the CompressedDocumentCount.
|
/// Gets or sets the CompressedDocumentCount.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressedDocumentCount { get; init; }
|
public long CompressedDocumentCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the BytesBeforeCompression.
|
/// Gets or sets the BytesBeforeCompression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesBeforeCompression { get; init; }
|
public long BytesBeforeCompression { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the BytesAfterCompression.
|
/// Gets or sets the BytesAfterCompression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesAfterCompression { get; init; }
|
public long BytesAfterCompression { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the CompressionCpuTicks.
|
/// Gets or sets the CompressionCpuTicks.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionCpuTicks { get; init; }
|
public long CompressionCpuTicks { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the DecompressionCpuTicks.
|
/// Gets or sets the DecompressionCpuTicks.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionCpuTicks { get; init; }
|
public long DecompressionCpuTicks { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the CompressionFailureCount.
|
/// Gets or sets the CompressionFailureCount.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionFailureCount { get; init; }
|
public long CompressionFailureCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the ChecksumFailureCount.
|
/// Gets or sets the ChecksumFailureCount.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long ChecksumFailureCount { get; init; }
|
public long ChecksumFailureCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the SafetyLimitRejectionCount.
|
/// Gets or sets the SafetyLimitRejectionCount.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long SafetyLimitRejectionCount { get; init; }
|
public long SafetyLimitRejectionCount { get; init; }
|
||||||
}
|
}
|
||||||
@@ -1,111 +1,109 @@
|
|||||||
using System.Threading;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Thread-safe counters for compression/decompression lifecycle events.
|
/// Thread-safe counters for compression/decompression lifecycle events.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompressionTelemetry
|
public sealed class CompressionTelemetry
|
||||||
{
|
{
|
||||||
|
private long _checksumFailureCount;
|
||||||
|
private long _compressedDocumentCount;
|
||||||
private long _compressionAttempts;
|
private long _compressionAttempts;
|
||||||
private long _compressionSuccesses;
|
private long _compressionCpuTicks;
|
||||||
private long _compressionFailures;
|
private long _compressionFailures;
|
||||||
private long _compressionSkippedTooSmall;
|
|
||||||
private long _compressionSkippedInsufficientSavings;
|
|
||||||
private long _decompressionAttempts;
|
|
||||||
private long _decompressionSuccesses;
|
|
||||||
private long _decompressionFailures;
|
|
||||||
private long _compressionInputBytes;
|
private long _compressionInputBytes;
|
||||||
private long _compressionOutputBytes;
|
private long _compressionOutputBytes;
|
||||||
private long _decompressionOutputBytes;
|
private long _compressionSkippedInsufficientSavings;
|
||||||
private long _compressedDocumentCount;
|
private long _compressionSkippedTooSmall;
|
||||||
private long _compressionCpuTicks;
|
private long _compressionSuccesses;
|
||||||
|
private long _decompressionAttempts;
|
||||||
private long _decompressionCpuTicks;
|
private long _decompressionCpuTicks;
|
||||||
private long _checksumFailureCount;
|
private long _decompressionFailures;
|
||||||
|
private long _decompressionOutputBytes;
|
||||||
|
private long _decompressionSuccesses;
|
||||||
private long _safetyLimitRejectionCount;
|
private long _safetyLimitRejectionCount;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of attempted compression operations.
|
/// Gets the number of attempted compression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionAttempts => Interlocked.Read(ref _compressionAttempts);
|
public long CompressionAttempts => Interlocked.Read(ref _compressionAttempts);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of successful compression operations.
|
/// Gets the number of successful compression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionSuccesses => Interlocked.Read(ref _compressionSuccesses);
|
public long CompressionSuccesses => Interlocked.Read(ref _compressionSuccesses);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of failed compression operations.
|
/// Gets the number of failed compression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionFailures => Interlocked.Read(ref _compressionFailures);
|
public long CompressionFailures => Interlocked.Read(ref _compressionFailures);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of compression attempts skipped because payloads were too small.
|
/// Gets the number of compression attempts skipped because payloads were too small.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionSkippedTooSmall => Interlocked.Read(ref _compressionSkippedTooSmall);
|
public long CompressionSkippedTooSmall => Interlocked.Read(ref _compressionSkippedTooSmall);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of compression attempts skipped due to insufficient savings.
|
/// Gets the number of compression attempts skipped due to insufficient savings.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionSkippedInsufficientSavings => Interlocked.Read(ref _compressionSkippedInsufficientSavings);
|
public long CompressionSkippedInsufficientSavings => Interlocked.Read(ref _compressionSkippedInsufficientSavings);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of attempted decompression operations.
|
/// Gets the number of attempted decompression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionAttempts => Interlocked.Read(ref _decompressionAttempts);
|
public long DecompressionAttempts => Interlocked.Read(ref _decompressionAttempts);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of successful decompression operations.
|
/// Gets the number of successful decompression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionSuccesses => Interlocked.Read(ref _decompressionSuccesses);
|
public long DecompressionSuccesses => Interlocked.Read(ref _decompressionSuccesses);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of failed decompression operations.
|
/// Gets the number of failed decompression operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionFailures => Interlocked.Read(ref _decompressionFailures);
|
public long DecompressionFailures => Interlocked.Read(ref _decompressionFailures);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total input bytes observed by compression attempts.
|
/// Gets the total input bytes observed by compression attempts.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionInputBytes => Interlocked.Read(ref _compressionInputBytes);
|
public long CompressionInputBytes => Interlocked.Read(ref _compressionInputBytes);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total output bytes produced by successful compression attempts.
|
/// Gets the total output bytes produced by successful compression attempts.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionOutputBytes => Interlocked.Read(ref _compressionOutputBytes);
|
public long CompressionOutputBytes => Interlocked.Read(ref _compressionOutputBytes);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total output bytes produced by successful decompression attempts.
|
/// Gets the total output bytes produced by successful decompression attempts.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionOutputBytes => Interlocked.Read(ref _decompressionOutputBytes);
|
public long DecompressionOutputBytes => Interlocked.Read(ref _decompressionOutputBytes);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of documents stored in compressed form.
|
/// Gets the number of documents stored in compressed form.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressedDocumentCount => Interlocked.Read(ref _compressedDocumentCount);
|
public long CompressedDocumentCount => Interlocked.Read(ref _compressedDocumentCount);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total CPU ticks spent on compression.
|
/// Gets the total CPU ticks spent on compression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressionCpuTicks => Interlocked.Read(ref _compressionCpuTicks);
|
public long CompressionCpuTicks => Interlocked.Read(ref _compressionCpuTicks);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total CPU ticks spent on decompression.
|
/// Gets the total CPU ticks spent on decompression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DecompressionCpuTicks => Interlocked.Read(ref _decompressionCpuTicks);
|
public long DecompressionCpuTicks => Interlocked.Read(ref _decompressionCpuTicks);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of checksum validation failures.
|
/// Gets the number of checksum validation failures.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long ChecksumFailureCount => Interlocked.Read(ref _checksumFailureCount);
|
public long ChecksumFailureCount => Interlocked.Read(ref _checksumFailureCount);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of decompression safety-limit rejections.
|
/// Gets the number of decompression safety-limit rejections.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long SafetyLimitRejectionCount => Interlocked.Read(ref _safetyLimitRejectionCount);
|
public long SafetyLimitRejectionCount => Interlocked.Read(ref _safetyLimitRejectionCount);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a compression attempt and its input byte size.
|
/// Records a compression attempt and its input byte size.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="inputBytes">The number of input bytes provided to compression.</param>
|
/// <param name="inputBytes">The number of input bytes provided to compression.</param>
|
||||||
public void RecordCompressionAttempt(int inputBytes)
|
public void RecordCompressionAttempt(int inputBytes)
|
||||||
@@ -115,7 +113,7 @@ public sealed class CompressionTelemetry
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a successful compression operation.
|
/// Records a successful compression operation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="outputBytes">The number of compressed bytes produced.</param>
|
/// <param name="outputBytes">The number of compressed bytes produced.</param>
|
||||||
public void RecordCompressionSuccess(int outputBytes)
|
public void RecordCompressionSuccess(int outputBytes)
|
||||||
@@ -126,49 +124,73 @@ public sealed class CompressionTelemetry
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a failed compression operation.
|
/// Records a failed compression operation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordCompressionFailure() => Interlocked.Increment(ref _compressionFailures);
|
public void RecordCompressionFailure()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _compressionFailures);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records that compression was skipped because the payload was too small.
|
/// Records that compression was skipped because the payload was too small.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordCompressionSkippedTooSmall() => Interlocked.Increment(ref _compressionSkippedTooSmall);
|
public void RecordCompressionSkippedTooSmall()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _compressionSkippedTooSmall);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records that compression was skipped due to insufficient expected savings.
|
/// Records that compression was skipped due to insufficient expected savings.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordCompressionSkippedInsufficientSavings() => Interlocked.Increment(ref _compressionSkippedInsufficientSavings);
|
public void RecordCompressionSkippedInsufficientSavings()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _compressionSkippedInsufficientSavings);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a decompression attempt.
|
/// Records a decompression attempt.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordDecompressionAttempt() => Interlocked.Increment(ref _decompressionAttempts);
|
public void RecordDecompressionAttempt()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _decompressionAttempts);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds CPU ticks spent performing compression.
|
/// Adds CPU ticks spent performing compression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ticks">The CPU ticks to add.</param>
|
/// <param name="ticks">The CPU ticks to add.</param>
|
||||||
public void RecordCompressionCpuTicks(long ticks) => Interlocked.Add(ref _compressionCpuTicks, ticks);
|
public void RecordCompressionCpuTicks(long ticks)
|
||||||
|
{
|
||||||
|
Interlocked.Add(ref _compressionCpuTicks, ticks);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds CPU ticks spent performing decompression.
|
/// Adds CPU ticks spent performing decompression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ticks">The CPU ticks to add.</param>
|
/// <param name="ticks">The CPU ticks to add.</param>
|
||||||
public void RecordDecompressionCpuTicks(long ticks) => Interlocked.Add(ref _decompressionCpuTicks, ticks);
|
public void RecordDecompressionCpuTicks(long ticks)
|
||||||
|
{
|
||||||
|
Interlocked.Add(ref _decompressionCpuTicks, ticks);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a checksum validation failure.
|
/// Records a checksum validation failure.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordChecksumFailure() => Interlocked.Increment(ref _checksumFailureCount);
|
public void RecordChecksumFailure()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _checksumFailureCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a decompression rejection due to safety limits.
|
/// Records a decompression rejection due to safety limits.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordSafetyLimitRejection() => Interlocked.Increment(ref _safetyLimitRejectionCount);
|
public void RecordSafetyLimitRejection()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _safetyLimitRejectionCount);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a successful decompression operation.
|
/// Records a successful decompression operation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="outputBytes">The number of decompressed bytes produced.</param>
|
/// <param name="outputBytes">The number of decompressed bytes produced.</param>
|
||||||
public void RecordDecompressionSuccess(int outputBytes)
|
public void RecordDecompressionSuccess(int outputBytes)
|
||||||
@@ -178,12 +200,15 @@ public sealed class CompressionTelemetry
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Records a failed decompression operation.
|
/// Records a failed decompression operation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void RecordDecompressionFailure() => Interlocked.Increment(ref _decompressionFailures);
|
public void RecordDecompressionFailure()
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _decompressionFailures);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns a point-in-time snapshot of compression telemetry.
|
/// Returns a point-in-time snapshot of compression telemetry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The aggregated compression statistics.</returns>
|
/// <returns>The aggregated compression statistics.</returns>
|
||||||
public CompressionStats GetSnapshot()
|
public CompressionStats GetSnapshot()
|
||||||
@@ -200,4 +225,4 @@ public sealed class CompressionTelemetry
|
|||||||
SafetyLimitRejectionCount = SafetyLimitRejectionCount
|
SafetyLimitRejectionCount = SafetyLimitRejectionCount
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,27 +3,27 @@ using System.IO.Compression;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
namespace ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Codec abstraction for payload compression and decompression.
|
/// Codec abstraction for payload compression and decompression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public interface ICompressionCodec
|
public interface ICompressionCodec
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Codec identifier.
|
/// Codec identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
CompressionCodec Codec { get; }
|
CompressionCodec Codec { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compresses input bytes.
|
/// Compresses input bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="input">Input payload bytes to compress.</param>
|
/// <param name="input">Input payload bytes to compress.</param>
|
||||||
/// <param name="level">Compression level to apply.</param>
|
/// <param name="level">Compression level to apply.</param>
|
||||||
byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level);
|
byte[] Compress(ReadOnlySpan<byte> input, CompressionLevel level);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Decompresses payload bytes with output bounds validation.
|
/// Decompresses payload bytes with output bounds validation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="input">Input payload bytes to decompress.</param>
|
/// <param name="input">Input payload bytes to decompress.</param>
|
||||||
/// <param name="expectedLength">Expected decompressed length.</param>
|
/// <param name="expectedLength">Expected decompressed length.</param>
|
||||||
/// <param name="maxDecompressedSizeBytes">Maximum allowed decompressed payload size in bytes.</param>
|
/// <param name="maxDecompressedSizeBytes">Maximum allowed decompressed payload size in bytes.</param>
|
||||||
byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes);
|
byte[] Decompress(ReadOnlySpan<byte> input, int expectedLength, int maxDecompressedSizeBytes);
|
||||||
}
|
}
|
||||||
@@ -1,52 +1,39 @@
|
|||||||
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Metadata;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
using ZB.MOM.WW.CBDD.Core.Metadata;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
|
||||||
using System.Threading;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core;
|
namespace ZB.MOM.WW.CBDD.Core;
|
||||||
|
|
||||||
internal interface ICompactionAwareCollection
|
internal interface ICompactionAwareCollection
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Refreshes index bindings after compaction.
|
/// Refreshes index bindings after compaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
void RefreshIndexBindingsAfterCompaction();
|
void RefreshIndexBindingsAfterCompaction();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Base class for database contexts.
|
/// Base class for database contexts.
|
||||||
/// Inherit and add DocumentCollection{T} properties for your entities.
|
/// Inherit and add DocumentCollection{T} properties for your entities.
|
||||||
/// Use partial class for Source Generator integration.
|
/// Use partial class for Source Generator integration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract partial class DocumentDbContext : IDisposable, ITransactionHolder
|
public abstract class DocumentDbContext : IDisposable, ITransactionHolder
|
||||||
{
|
{
|
||||||
|
internal readonly ChangeStreamDispatcher _cdc;
|
||||||
|
private readonly List<ICompactionAwareCollection> _compactionAwareCollections = new();
|
||||||
|
|
||||||
|
private readonly IReadOnlyDictionary<Type, object> _model;
|
||||||
|
private readonly List<IDocumentMapper> _registeredMappers = new();
|
||||||
private readonly IStorageEngine _storage;
|
private readonly IStorageEngine _storage;
|
||||||
internal readonly CDC.ChangeStreamDispatcher _cdc;
|
private readonly SemaphoreSlim _transactionLock = new(1, 1);
|
||||||
protected bool _disposed;
|
protected bool _disposed;
|
||||||
private readonly SemaphoreSlim _transactionLock = new SemaphoreSlim(1, 1);
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current active transaction, if any.
|
/// Creates a new database context with default configuration
|
||||||
/// </summary>
|
|
||||||
public ITransaction? CurrentTransaction
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (_disposed)
|
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
|
||||||
return field != null && (field.State == TransactionState.Active) ? field : null;
|
|
||||||
}
|
|
||||||
private set;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Creates a new database context with default configuration
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="databasePath">The database file path.</param>
|
/// <param name="databasePath">The database file path.</param>
|
||||||
protected DocumentDbContext(string databasePath)
|
protected DocumentDbContext(string databasePath)
|
||||||
@@ -55,7 +42,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new database context with default storage configuration and custom compression settings.
|
/// Creates a new database context with default storage configuration and custom compression settings.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="databasePath">The database file path.</param>
|
/// <param name="databasePath">The database file path.</param>
|
||||||
/// <param name="compressionOptions">Compression behavior options.</param>
|
/// <param name="compressionOptions">Compression behavior options.</param>
|
||||||
@@ -65,7 +52,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new database context with custom configuration
|
/// Creates a new database context with custom configuration
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="databasePath">The database file path.</param>
|
/// <param name="databasePath">The database file path.</param>
|
||||||
/// <param name="config">The page file configuration.</param>
|
/// <param name="config">The page file configuration.</param>
|
||||||
@@ -75,7 +62,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new database context with custom storage and compression configuration.
|
/// Creates a new database context with custom storage and compression configuration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="databasePath">The database file path.</param>
|
/// <param name="databasePath">The database file path.</param>
|
||||||
/// <param name="config">The page file configuration.</param>
|
/// <param name="config">The page file configuration.</param>
|
||||||
@@ -91,7 +78,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
throw new ArgumentNullException(nameof(databasePath));
|
throw new ArgumentNullException(nameof(databasePath));
|
||||||
|
|
||||||
_storage = new StorageEngine(databasePath, config, compressionOptions, maintenanceOptions);
|
_storage = new StorageEngine(databasePath, config, compressionOptions, maintenanceOptions);
|
||||||
_cdc = new CDC.ChangeStreamDispatcher();
|
_cdc = new ChangeStreamDispatcher();
|
||||||
_storage.RegisterCdc(_cdc);
|
_storage.RegisterCdc(_cdc);
|
||||||
|
|
||||||
// Initialize model before collections
|
// Initialize model before collections
|
||||||
@@ -102,108 +89,41 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes document collections for the context.
|
/// Gets the current active transaction, if any.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected virtual void InitializeCollections()
|
public ITransaction? CurrentTransaction
|
||||||
{
|
{
|
||||||
// Derived classes can override to initialize collections
|
get
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
return field != null && field.State == TransactionState.Active ? field : null;
|
||||||
|
}
|
||||||
|
private set;
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly IReadOnlyDictionary<Type, object> _model;
|
|
||||||
private readonly List<IDocumentMapper> _registeredMappers = new();
|
|
||||||
private readonly List<ICompactionAwareCollection> _compactionAwareCollections = new();
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the concrete storage engine for advanced scenarios in derived contexts.
|
/// Gets the concrete storage engine for advanced scenarios in derived contexts.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected StorageEngine Engine => (StorageEngine)_storage;
|
protected StorageEngine Engine => (StorageEngine)_storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets compression options bound to this context's storage engine.
|
/// Gets compression options bound to this context's storage engine.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected CompressionOptions CompressionOptions => _storage.CompressionOptions;
|
protected CompressionOptions CompressionOptions => _storage.CompressionOptions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the compression service for codec operations.
|
/// Gets the compression service for codec operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected CompressionService CompressionService => _storage.CompressionService;
|
protected CompressionService CompressionService => _storage.CompressionService;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets compression telemetry counters.
|
/// Gets compression telemetry counters.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
protected CompressionTelemetry CompressionTelemetry => _storage.CompressionTelemetry;
|
protected CompressionTelemetry CompressionTelemetry => _storage.CompressionTelemetry;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Override to configure the model using Fluent API.
|
/// Releases resources used by the context.
|
||||||
/// </summary>
|
|
||||||
/// <param name="modelBuilder">The model builder instance.</param>
|
|
||||||
protected virtual void OnModelCreating(ModelBuilder modelBuilder)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Helper to create a DocumentCollection instance with custom TId.
|
|
||||||
/// Used by derived classes in InitializeCollections for typed primary keys.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TId">The document identifier type.</typeparam>
|
|
||||||
/// <typeparam name="T">The document type.</typeparam>
|
|
||||||
/// <param name="mapper">The mapper used for document serialization and key access.</param>
|
|
||||||
/// <returns>The created document collection.</returns>
|
|
||||||
protected DocumentCollection<TId, T> CreateCollection<TId, T>(IDocumentMapper<TId, T> mapper)
|
|
||||||
where T : class
|
|
||||||
{
|
|
||||||
if (_disposed)
|
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
|
||||||
|
|
||||||
string? customName = null;
|
|
||||||
EntityTypeBuilder<T>? builder = null;
|
|
||||||
|
|
||||||
if (_model.TryGetValue(typeof(T), out var builderObj))
|
|
||||||
{
|
|
||||||
builder = builderObj as EntityTypeBuilder<T>;
|
|
||||||
customName = builder?.CollectionName;
|
|
||||||
}
|
|
||||||
|
|
||||||
_registeredMappers.Add(mapper);
|
|
||||||
var collection = new DocumentCollection<TId, T>(_storage, this, mapper, customName);
|
|
||||||
if (collection is ICompactionAwareCollection compactionAwareCollection)
|
|
||||||
{
|
|
||||||
_compactionAwareCollections.Add(compactionAwareCollection);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Apply configurations from ModelBuilder
|
|
||||||
if (builder != null)
|
|
||||||
{
|
|
||||||
foreach (var indexBuilder in builder.Indexes)
|
|
||||||
{
|
|
||||||
collection.ApplyIndexBuilder(indexBuilder);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
_storage.RegisterMappers(_registeredMappers);
|
|
||||||
|
|
||||||
return collection;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the document collection for the specified entity type using an ObjectId as the key.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="T">The type of entity to retrieve the document collection for. Must be a reference type.</typeparam>
|
|
||||||
/// <returns>A DocumentCollection<ObjectId, T> instance for the specified entity type.</returns>
|
|
||||||
public DocumentCollection<ObjectId, T> Set<T>() where T : class => Set<ObjectId, T>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets a collection for managing documents of type T, identified by keys of type TId.
|
|
||||||
/// Override is generated automatically by the Source Generator for partial DbContext classes.
|
|
||||||
/// </summary>
|
|
||||||
/// <typeparam name="TId">The type of the unique identifier for documents in the collection.</typeparam>
|
|
||||||
/// <typeparam name="T">The type of the document to be managed. Must be a reference type.</typeparam>
|
|
||||||
/// <returns>A DocumentCollection<TId, T> instance for performing operations on documents of type T.</returns>
|
|
||||||
public virtual DocumentCollection<TId, T> Set<TId, T>() where T : class
|
|
||||||
=> throw new InvalidOperationException($"No collection registered for entity type '{typeof(T).Name}' with key type '{typeof(TId).Name}'.");
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Releases resources used by the context.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
@@ -220,7 +140,102 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins a transaction or returns the current active transaction.
|
/// Gets the current active transaction or starts a new one.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The active transaction.</returns>
|
||||||
|
public ITransaction GetCurrentTransactionOrStart()
|
||||||
|
{
|
||||||
|
return BeginTransaction();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the current active transaction or starts a new one asynchronously.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The active transaction.</returns>
|
||||||
|
public async Task<ITransaction> GetCurrentTransactionOrStartAsync()
|
||||||
|
{
|
||||||
|
return await BeginTransactionAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Initializes document collections for the context.
|
||||||
|
/// </summary>
|
||||||
|
protected virtual void InitializeCollections()
|
||||||
|
{
|
||||||
|
// Derived classes can override to initialize collections
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Override to configure the model using Fluent API.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="modelBuilder">The model builder instance.</param>
|
||||||
|
protected virtual void OnModelCreating(ModelBuilder modelBuilder)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Helper to create a DocumentCollection instance with custom TId.
|
||||||
|
/// Used by derived classes in InitializeCollections for typed primary keys.
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="TId">The document identifier type.</typeparam>
|
||||||
|
/// <typeparam name="T">The document type.</typeparam>
|
||||||
|
/// <param name="mapper">The mapper used for document serialization and key access.</param>
|
||||||
|
/// <returns>The created document collection.</returns>
|
||||||
|
protected DocumentCollection<TId, T> CreateCollection<TId, T>(IDocumentMapper<TId, T> mapper)
|
||||||
|
where T : class
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
|
string? customName = null;
|
||||||
|
EntityTypeBuilder<T>? builder = null;
|
||||||
|
|
||||||
|
if (_model.TryGetValue(typeof(T), out object? builderObj))
|
||||||
|
{
|
||||||
|
builder = builderObj as EntityTypeBuilder<T>;
|
||||||
|
customName = builder?.CollectionName;
|
||||||
|
}
|
||||||
|
|
||||||
|
_registeredMappers.Add(mapper);
|
||||||
|
var collection = new DocumentCollection<TId, T>(_storage, this, mapper, customName);
|
||||||
|
if (collection is ICompactionAwareCollection compactionAwareCollection)
|
||||||
|
_compactionAwareCollections.Add(compactionAwareCollection);
|
||||||
|
|
||||||
|
// Apply configurations from ModelBuilder
|
||||||
|
if (builder != null)
|
||||||
|
foreach (var indexBuilder in builder.Indexes)
|
||||||
|
collection.ApplyIndexBuilder(indexBuilder);
|
||||||
|
|
||||||
|
_storage.RegisterMappers(_registeredMappers);
|
||||||
|
|
||||||
|
return collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the document collection for the specified entity type using an ObjectId as the key.
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="T">The type of entity to retrieve the document collection for. Must be a reference type.</typeparam>
|
||||||
|
/// <returns>A DocumentCollection<ObjectId, T> instance for the specified entity type.</returns>
|
||||||
|
public DocumentCollection<ObjectId, T> Set<T>() where T : class
|
||||||
|
{
|
||||||
|
return Set<ObjectId, T>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a collection for managing documents of type T, identified by keys of type TId.
|
||||||
|
/// Override is generated automatically by the Source Generator for partial DbContext classes.
|
||||||
|
/// </summary>
|
||||||
|
/// <typeparam name="TId">The type of the unique identifier for documents in the collection.</typeparam>
|
||||||
|
/// <typeparam name="T">The type of the document to be managed. Must be a reference type.</typeparam>
|
||||||
|
/// <returns>A DocumentCollection<TId, T> instance for performing operations on documents of type T.</returns>
|
||||||
|
public virtual DocumentCollection<TId, T> Set<TId, T>() where T : class
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
$"No collection registered for entity type '{typeof(T).Name}' with key type '{typeof(TId).Name}'.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Begins a transaction or returns the current active transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The active transaction.</returns>
|
/// <returns>The active transaction.</returns>
|
||||||
public ITransaction BeginTransaction()
|
public ITransaction BeginTransaction()
|
||||||
@@ -243,7 +258,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins a transaction asynchronously or returns the current active transaction.
|
/// Begins a transaction asynchronously or returns the current active transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
/// <returns>The active transaction.</returns>
|
/// <returns>The active transaction.</returns>
|
||||||
@@ -252,7 +267,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
if (_disposed)
|
if (_disposed)
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
bool lockAcquired = false;
|
var lockAcquired = false;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await _transactionLock.WaitAsync(ct);
|
await _transactionLock.WaitAsync(ct);
|
||||||
@@ -271,32 +286,13 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current active transaction or starts a new one.
|
/// Commits the current transaction if one is active.
|
||||||
/// </summary>
|
|
||||||
/// <returns>The active transaction.</returns>
|
|
||||||
public ITransaction GetCurrentTransactionOrStart()
|
|
||||||
{
|
|
||||||
return BeginTransaction();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the current active transaction or starts a new one asynchronously.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>The active transaction.</returns>
|
|
||||||
public async Task<ITransaction> GetCurrentTransactionOrStartAsync()
|
|
||||||
{
|
|
||||||
return await BeginTransactionAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Commits the current transaction if one is active.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void SaveChanges()
|
public void SaveChanges()
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
if (CurrentTransaction != null)
|
if (CurrentTransaction != null)
|
||||||
{
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
CurrentTransaction.Commit();
|
CurrentTransaction.Commit();
|
||||||
@@ -305,19 +301,17 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
{
|
{
|
||||||
CurrentTransaction = null;
|
CurrentTransaction = null;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits the current transaction asynchronously if one is active.
|
/// Commits the current transaction asynchronously if one is active.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
public async Task SaveChangesAsync(CancellationToken ct = default)
|
public async Task SaveChangesAsync(CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
if (CurrentTransaction != null)
|
if (CurrentTransaction != null)
|
||||||
{
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await CurrentTransaction.CommitAsync(ct);
|
await CurrentTransaction.CommitAsync(ct);
|
||||||
@@ -325,40 +319,40 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
CurrentTransaction = null;
|
CurrentTransaction = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Executes a checkpoint using the requested mode.
|
||||||
/// Executes a checkpoint using the requested mode.
|
/// </summary>
|
||||||
/// </summary>
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
/// <returns>The checkpoint execution result.</returns>
|
public CheckpointResult Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)
|
||||||
public CheckpointResult Checkpoint(CheckpointMode mode = CheckpointMode.Truncate)
|
{
|
||||||
{
|
if (_disposed)
|
||||||
if (_disposed)
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
|
||||||
|
return Engine.Checkpoint(mode);
|
||||||
return Engine.Checkpoint(mode);
|
}
|
||||||
}
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Executes a checkpoint asynchronously using the requested mode.
|
||||||
/// Executes a checkpoint asynchronously using the requested mode.
|
/// </summary>
|
||||||
/// </summary>
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
/// <returns>The checkpoint execution result.</returns>
|
public Task<CheckpointResult> CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate,
|
||||||
public Task<CheckpointResult> CheckpointAsync(CheckpointMode mode = CheckpointMode.Truncate, CancellationToken ct = default)
|
CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
return Engine.CheckpointAsync(mode, ct);
|
return Engine.CheckpointAsync(mode, ct);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionStats GetCompressionStats()
|
public CompressionStats GetCompressionStats()
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
@@ -368,7 +362,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compaction execution options.</param>
|
/// <param name="options">Compaction execution options.</param>
|
||||||
public CompactionStats Compact(CompactionOptions? options = null)
|
public CompactionStats Compact(CompactionOptions? options = null)
|
||||||
@@ -382,7 +376,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
/// Runs offline compaction by default. Set options to online mode for a bounded online pass.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compaction execution options.</param>
|
/// <param name="options">Compaction execution options.</param>
|
||||||
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
||||||
@@ -395,7 +389,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Alias for <see cref="Compact(CompactionOptions?)"/>.
|
/// Alias for <see cref="Compact(CompactionOptions?)" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compaction execution options.</param>
|
/// <param name="options">Compaction execution options.</param>
|
||||||
public CompactionStats Vacuum(CompactionOptions? options = null)
|
public CompactionStats Vacuum(CompactionOptions? options = null)
|
||||||
@@ -409,7 +403,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Async alias for <see cref="CompactAsync(CompactionOptions?, CancellationToken)"/>.
|
/// Async alias for <see cref="CompactAsync(CompactionOptions?, CancellationToken)" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compaction execution options.</param>
|
/// <param name="options">Compaction execution options.</param>
|
||||||
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
||||||
@@ -437,14 +431,11 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
|
|
||||||
private void RefreshCollectionBindingsAfterCompaction()
|
private void RefreshCollectionBindingsAfterCompaction()
|
||||||
{
|
{
|
||||||
foreach (var collection in _compactionAwareCollections)
|
foreach (var collection in _compactionAwareCollections) collection.RefreshIndexBindingsAfterCompaction();
|
||||||
{
|
|
||||||
collection.RefreshIndexBindingsAfterCompaction();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets page usage grouped by page type.
|
/// Gets page usage grouped by page type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
||||||
{
|
{
|
||||||
@@ -455,7 +446,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets per-collection page usage diagnostics.
|
/// Gets per-collection page usage diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
||||||
{
|
{
|
||||||
@@ -466,7 +457,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets per-collection compression ratio diagnostics.
|
/// Gets per-collection compression ratio diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
||||||
{
|
{
|
||||||
@@ -477,7 +468,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets free-list summary diagnostics.
|
/// Gets free-list summary diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public FreeListSummary GetFreeListSummary()
|
public FreeListSummary GetFreeListSummary()
|
||||||
{
|
{
|
||||||
@@ -488,7 +479,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets page-level fragmentation diagnostics.
|
/// Gets page-level fragmentation diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public FragmentationMapReport GetFragmentationMap()
|
public FragmentationMapReport GetFragmentationMap()
|
||||||
{
|
{
|
||||||
@@ -499,7 +490,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Runs compression migration as dry-run estimation by default.
|
/// Runs compression migration as dry-run estimation by default.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compression migration options.</param>
|
/// <param name="options">Compression migration options.</param>
|
||||||
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
||||||
@@ -511,15 +502,16 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Runs compression migration asynchronously as dry-run estimation by default.
|
/// Runs compression migration asynchronously as dry-run estimation by default.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Compression migration options.</param>
|
/// <param name="options">Compression migration options.</param>
|
||||||
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
/// <param name="ct">Cancellation token for the asynchronous operation.</param>
|
||||||
public Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null, CancellationToken ct = default)
|
public Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null,
|
||||||
|
CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
throw new ObjectDisposedException(nameof(DocumentDbContext));
|
||||||
|
|
||||||
return Engine.MigrateCompressionAsync(options, ct);
|
return Engine.MigrateCompressionAsync(options, ct);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,131 +1,129 @@
|
|||||||
using System.Buffers;
|
using System.Buffers;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
internal sealed class BTreeCursor : IBTreeCursor
|
internal sealed class BTreeCursor : IBTreeCursor
|
||||||
{
|
{
|
||||||
private readonly BTreeIndex _index;
|
private readonly List<IndexEntry> _currentEntries;
|
||||||
private readonly ulong _transactionId;
|
private readonly BTreeIndex _index;
|
||||||
private readonly IIndexStorage _storage;
|
private readonly IIndexStorage _storage;
|
||||||
|
private readonly ulong _transactionId;
|
||||||
// State
|
|
||||||
private byte[] _pageBuffer;
|
|
||||||
private uint _currentPageId;
|
|
||||||
private int _currentEntryIndex;
|
private int _currentEntryIndex;
|
||||||
private BTreeNodeHeader _currentHeader;
|
private BTreeNodeHeader _currentHeader;
|
||||||
private List<IndexEntry> _currentEntries;
|
private uint _currentPageId;
|
||||||
private bool _isValid;
|
private bool _isValid;
|
||||||
|
|
||||||
/// <summary>
|
// State
|
||||||
/// Initializes a new instance of the <see cref="BTreeCursor"/> class.
|
private byte[] _pageBuffer;
|
||||||
/// </summary>
|
|
||||||
/// <param name="index">The index to traverse.</param>
|
/// <summary>
|
||||||
/// <param name="storage">The storage engine for page access.</param>
|
/// Initializes a new instance of the <see cref="BTreeCursor" /> class.
|
||||||
/// <param name="transactionId">The transaction identifier used for reads.</param>
|
/// </summary>
|
||||||
public BTreeCursor(BTreeIndex index, IIndexStorage storage, ulong transactionId)
|
/// <param name="index">The index to traverse.</param>
|
||||||
{
|
/// <param name="storage">The storage engine for page access.</param>
|
||||||
_index = index;
|
/// <param name="transactionId">The transaction identifier used for reads.</param>
|
||||||
_storage = storage;
|
public BTreeCursor(BTreeIndex index, IIndexStorage storage, ulong transactionId)
|
||||||
|
{
|
||||||
|
_index = index;
|
||||||
|
_storage = storage;
|
||||||
_transactionId = transactionId;
|
_transactionId = transactionId;
|
||||||
_pageBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
_pageBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||||
_currentEntries = new List<IndexEntry>();
|
_currentEntries = new List<IndexEntry>();
|
||||||
_isValid = false;
|
_isValid = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current index entry at the cursor position.
|
/// Gets the current index entry at the cursor position.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexEntry Current
|
public IndexEntry Current
|
||||||
{
|
{
|
||||||
get
|
get
|
||||||
{
|
{
|
||||||
if (!_isValid) throw new InvalidOperationException("Cursor is not valid.");
|
if (!_isValid) throw new InvalidOperationException("Cursor is not valid.");
|
||||||
return _currentEntries[_currentEntryIndex];
|
return _currentEntries[_currentEntryIndex];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the first entry in the index.
|
/// Moves the cursor to the first entry in the index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns><see langword="true"/> if an entry is available; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if an entry is available; otherwise, <see langword="false" />.</returns>
|
||||||
public bool MoveToFirst()
|
public bool MoveToFirst()
|
||||||
{
|
{
|
||||||
// Find left-most leaf
|
// Find left-most leaf
|
||||||
var pageId = _index.RootPageId;
|
uint pageId = _index.RootPageId;
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
LoadPage(pageId);
|
LoadPage(pageId);
|
||||||
if (_currentHeader.IsLeaf) break;
|
if (_currentHeader.IsLeaf) break;
|
||||||
|
|
||||||
// Go to first child (P0)
|
// Go to first child (P0)
|
||||||
// Internal node format: [Header] [P0] [Entry1] ...
|
// Internal node format: [Header] [P0] [Entry1] ...
|
||||||
var dataOffset = 32 + 20;
|
int dataOffset = 32 + 20;
|
||||||
pageId = BitConverter.ToUInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
pageId = BitConverter.ToUInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
||||||
}
|
}
|
||||||
|
|
||||||
return PositionAtStart();
|
return PositionAtStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the last entry in the index.
|
/// Moves the cursor to the last entry in the index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns><see langword="true"/> if an entry is available; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if an entry is available; otherwise, <see langword="false" />.</returns>
|
||||||
public bool MoveToLast()
|
public bool MoveToLast()
|
||||||
{
|
{
|
||||||
// Find right-most leaf
|
// Find right-most leaf
|
||||||
var pageId = _index.RootPageId;
|
uint pageId = _index.RootPageId;
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
LoadPage(pageId);
|
LoadPage(pageId);
|
||||||
if (_currentHeader.IsLeaf) break;
|
if (_currentHeader.IsLeaf) break;
|
||||||
|
|
||||||
// Go to last child (last pointer)
|
|
||||||
// Iterate all entries to find last pointer
|
|
||||||
// P0 is at 32+20 (4 bytes). Entry 0 starts at 32+20+4.
|
|
||||||
|
|
||||||
// Wait, we need the last pointer.
|
|
||||||
// P0 is at offset.
|
|
||||||
// Then EncryCount entries: Key + Pointer.
|
|
||||||
// We want the last pointer.
|
|
||||||
|
|
||||||
// Re-read P0 just in case
|
|
||||||
uint lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(32 + 20, 4));
|
|
||||||
|
|
||||||
var offset = 32 + 20 + 4;
|
// Go to last child (last pointer)
|
||||||
for (int i = 0; i < _currentHeader.EntryCount; i++)
|
// Iterate all entries to find last pointer
|
||||||
|
// P0 is at 32+20 (4 bytes). Entry 0 starts at 32+20+4.
|
||||||
|
|
||||||
|
// Wait, we need the last pointer.
|
||||||
|
// P0 is at offset.
|
||||||
|
// Then EncryCount entries: Key + Pointer.
|
||||||
|
// We want the last pointer.
|
||||||
|
|
||||||
|
// Re-read P0 just in case
|
||||||
|
var lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(32 + 20, 4));
|
||||||
|
|
||||||
|
int offset = 32 + 20 + 4;
|
||||||
|
for (var i = 0; i < _currentHeader.EntryCount; i++)
|
||||||
{
|
{
|
||||||
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(offset, 4));
|
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(offset, 4));
|
||||||
offset += 4 + keyLen;
|
offset += 4 + keyLen;
|
||||||
lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(offset, 4));
|
lastPointer = BitConverter.ToUInt32(_pageBuffer.AsSpan(offset, 4));
|
||||||
offset += 4;
|
offset += 4;
|
||||||
}
|
}
|
||||||
|
|
||||||
pageId = lastPointer;
|
pageId = lastPointer;
|
||||||
}
|
}
|
||||||
|
|
||||||
return PositionAtEnd();
|
return PositionAtEnd();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Seeks to the specified key or the next greater key.
|
/// Seeks to the specified key or the next greater key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The key to seek.</param>
|
/// <param name="key">The key to seek.</param>
|
||||||
/// <returns>
|
/// <returns>
|
||||||
/// <see langword="true"/> if an exact key match is found; otherwise, <see langword="false"/>.
|
/// <see langword="true" /> if an exact key match is found; otherwise, <see langword="false" />.
|
||||||
/// </returns>
|
/// </returns>
|
||||||
public bool Seek(IndexKey key)
|
public bool Seek(IndexKey key)
|
||||||
{
|
{
|
||||||
// Use Index to find leaf
|
// Use Index to find leaf
|
||||||
var leafPageId = _index.FindLeafNode(key, _transactionId);
|
uint leafPageId = _index.FindLeafNode(key, _transactionId);
|
||||||
LoadPage(leafPageId);
|
LoadPage(leafPageId);
|
||||||
ParseEntries();
|
ParseEntries();
|
||||||
|
|
||||||
// Binary search in entries
|
// Binary search in entries
|
||||||
var idx = _currentEntries.BinarySearch(new IndexEntry(key, default(DocumentLocation)));
|
int idx = _currentEntries.BinarySearch(new IndexEntry(key, default(DocumentLocation)));
|
||||||
|
|
||||||
if (idx >= 0)
|
if (idx >= 0)
|
||||||
{
|
{
|
||||||
// Found exact match
|
// Found exact match
|
||||||
@@ -133,51 +131,44 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
_isValid = true;
|
_isValid = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
// Not found, ~idx is the next larger value
|
||||||
|
_currentEntryIndex = ~idx;
|
||||||
|
|
||||||
|
if (_currentEntryIndex < _currentEntries.Count)
|
||||||
{
|
{
|
||||||
// Not found, ~idx is the next larger value
|
_isValid = true;
|
||||||
_currentEntryIndex = ~idx;
|
return false; // Positioned at next greater
|
||||||
|
}
|
||||||
if (_currentEntryIndex < _currentEntries.Count)
|
|
||||||
|
// Key is larger than max in this page, move to next page
|
||||||
|
if (_currentHeader.NextLeafPageId != 0)
|
||||||
|
{
|
||||||
|
LoadPage(_currentHeader.NextLeafPageId);
|
||||||
|
ParseEntries();
|
||||||
|
_currentEntryIndex = 0;
|
||||||
|
if (_currentEntries.Count > 0)
|
||||||
{
|
{
|
||||||
_isValid = true;
|
_isValid = true;
|
||||||
return false; // Positioned at next greater
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Key is larger than max in this page, move to next page
|
|
||||||
if (_currentHeader.NextLeafPageId != 0)
|
|
||||||
{
|
|
||||||
LoadPage(_currentHeader.NextLeafPageId);
|
|
||||||
ParseEntries();
|
|
||||||
_currentEntryIndex = 0;
|
|
||||||
if (_currentEntries.Count > 0)
|
|
||||||
{
|
|
||||||
_isValid = true;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// End of index
|
|
||||||
_isValid = false;
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
// End of index
|
||||||
/// <summary>
|
_isValid = false;
|
||||||
/// Moves the cursor to the next entry.
|
return false;
|
||||||
/// </summary>
|
}
|
||||||
/// <returns><see langword="true"/> if the cursor moved to a valid entry; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public bool MoveNext()
|
/// <summary>
|
||||||
{
|
/// Moves the cursor to the next entry.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns><see langword="true" /> if the cursor moved to a valid entry; otherwise, <see langword="false" />.</returns>
|
||||||
|
public bool MoveNext()
|
||||||
|
{
|
||||||
if (!_isValid) return false;
|
if (!_isValid) return false;
|
||||||
|
|
||||||
_currentEntryIndex++;
|
_currentEntryIndex++;
|
||||||
if (_currentEntryIndex < _currentEntries.Count)
|
if (_currentEntryIndex < _currentEntries.Count) return true;
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move to next page
|
// Move to next page
|
||||||
if (_currentHeader.NextLeafPageId != 0)
|
if (_currentHeader.NextLeafPageId != 0)
|
||||||
@@ -186,23 +177,20 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
return PositionAtStart();
|
return PositionAtStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
_isValid = false;
|
_isValid = false;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the previous entry.
|
/// Moves the cursor to the previous entry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns><see langword="true"/> if the cursor moved to a valid entry; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the cursor moved to a valid entry; otherwise, <see langword="false" />.</returns>
|
||||||
public bool MovePrev()
|
public bool MovePrev()
|
||||||
{
|
{
|
||||||
if (!_isValid) return false;
|
if (!_isValid) return false;
|
||||||
|
|
||||||
_currentEntryIndex--;
|
_currentEntryIndex--;
|
||||||
if (_currentEntryIndex >= 0)
|
if (_currentEntryIndex >= 0) return true;
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move to prev page
|
// Move to prev page
|
||||||
if (_currentHeader.PrevLeafPageId != 0)
|
if (_currentHeader.PrevLeafPageId != 0)
|
||||||
@@ -211,9 +199,21 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
return PositionAtEnd();
|
return PositionAtEnd();
|
||||||
}
|
}
|
||||||
|
|
||||||
_isValid = false;
|
_isValid = false;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Releases cursor resources.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (_pageBuffer != null)
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(_pageBuffer);
|
||||||
|
_pageBuffer = null!;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void LoadPage(uint pageId)
|
private void LoadPage(uint pageId)
|
||||||
{
|
{
|
||||||
@@ -229,9 +229,9 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
// Helper to parse entries from current page buffer
|
// Helper to parse entries from current page buffer
|
||||||
// (Similar to BTreeIndex.ReadLeafEntries)
|
// (Similar to BTreeIndex.ReadLeafEntries)
|
||||||
_currentEntries.Clear();
|
_currentEntries.Clear();
|
||||||
var dataOffset = 32 + 20;
|
int dataOffset = 32 + 20;
|
||||||
|
|
||||||
for (int i = 0; i < _currentHeader.EntryCount; i++)
|
for (var i = 0; i < _currentHeader.EntryCount; i++)
|
||||||
{
|
{
|
||||||
// Read Key
|
// Read Key
|
||||||
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
var keyLen = BitConverter.ToInt32(_pageBuffer.AsSpan(dataOffset, 4));
|
||||||
@@ -257,12 +257,10 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
_isValid = true;
|
_isValid = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
// Empty page? Should not happen in helper logic unless root leaf is empty
|
||||||
// Empty page? Should not happen in helper logic unless root leaf is empty
|
_isValid = false;
|
||||||
_isValid = false;
|
return false;
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool PositionAtEnd()
|
private bool PositionAtEnd()
|
||||||
@@ -274,22 +272,8 @@ internal sealed class BTreeCursor : IBTreeCursor
|
|||||||
_isValid = true;
|
_isValid = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
_isValid = false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
_isValid = false;
|
||||||
/// Releases cursor resources.
|
return false;
|
||||||
/// </summary>
|
|
||||||
public void Dispose()
|
|
||||||
{
|
|
||||||
if (_pageBuffer != null)
|
|
||||||
{
|
|
||||||
ArrayPool<byte>.Shared.Return(_pageBuffer);
|
|
||||||
_pageBuffer = null!;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,27 +1,26 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using System;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Represents an entry in an index mapping a key to a document location.
|
||||||
/// Represents an entry in an index mapping a key to a document location.
|
/// Implemented as struct for memory efficiency.
|
||||||
/// Implemented as struct for memory efficiency.
|
/// </summary>
|
||||||
/// </summary>
|
|
||||||
public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the index key.
|
/// Gets or sets the index key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexKey Key { get; set; }
|
public IndexKey Key { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the document location for the key.
|
/// Gets or sets the document location for the key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DocumentLocation Location { get; set; }
|
public DocumentLocation Location { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexEntry"/> struct.
|
/// Initializes a new instance of the <see cref="IndexEntry" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <param name="location">The document location.</param>
|
/// <param name="location">The document location.</param>
|
||||||
@@ -34,7 +33,7 @@ public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
|||||||
// Backward compatibility: constructor that takes ObjectId (for migration)
|
// Backward compatibility: constructor that takes ObjectId (for migration)
|
||||||
// Will be removed once all code is migrated
|
// Will be removed once all code is migrated
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a legacy instance of the <see cref="IndexEntry"/> struct for migration scenarios.
|
/// Initializes a legacy instance of the <see cref="IndexEntry" /> struct for migration scenarios.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <param name="documentId">The legacy document identifier.</param>
|
/// <param name="documentId">The legacy document identifier.</param>
|
||||||
@@ -47,12 +46,12 @@ public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compares this entry to another entry by key.
|
/// Compares this entry to another entry by key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The other index entry to compare.</param>
|
/// <param name="other">The other index entry to compare.</param>
|
||||||
/// <returns>
|
/// <returns>
|
||||||
/// A value less than zero if this instance is less than <paramref name="other"/>,
|
/// A value less than zero if this instance is less than <paramref name="other" />,
|
||||||
/// zero if they are equal, or greater than zero if this instance is greater.
|
/// zero if they are equal, or greater than zero if this instance is greater.
|
||||||
/// </returns>
|
/// </returns>
|
||||||
public int CompareTo(IndexEntry other)
|
public int CompareTo(IndexEntry other)
|
||||||
{
|
{
|
||||||
@@ -60,76 +59,76 @@ public struct IndexEntry : IComparable<IndexEntry>, IComparable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compares this entry to another object.
|
/// Compares this entry to another object.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="obj">The object to compare.</param>
|
/// <param name="obj">The object to compare.</param>
|
||||||
/// <returns>
|
/// <returns>
|
||||||
/// A value less than zero if this instance is less than <paramref name="obj"/>,
|
/// A value less than zero if this instance is less than <paramref name="obj" />,
|
||||||
/// zero if they are equal, or greater than zero if this instance is greater.
|
/// zero if they are equal, or greater than zero if this instance is greater.
|
||||||
/// </returns>
|
/// </returns>
|
||||||
/// <exception cref="ArgumentException">Thrown when <paramref name="obj"/> is not an <see cref="IndexEntry"/>.</exception>
|
/// <exception cref="ArgumentException">Thrown when <paramref name="obj" /> is not an <see cref="IndexEntry" />.</exception>
|
||||||
public int CompareTo(object? obj)
|
public int CompareTo(object? obj)
|
||||||
{
|
{
|
||||||
if (obj is IndexEntry other) return CompareTo(other);
|
if (obj is IndexEntry other) return CompareTo(other);
|
||||||
throw new ArgumentException("Object is not an IndexEntry");
|
throw new ArgumentException("Object is not an IndexEntry");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// B+Tree node for index storage.
|
/// B+Tree node for index storage.
|
||||||
/// Uses struct for node metadata to minimize allocations.
|
/// Uses struct for node metadata to minimize allocations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public struct BTreeNodeHeader
|
public struct BTreeNodeHeader
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the page identifier.
|
/// Gets or sets the page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageId { get; set; }
|
public uint PageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets a value indicating whether this node is a leaf node.
|
/// Gets or sets a value indicating whether this node is a leaf node.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsLeaf { get; set; }
|
public bool IsLeaf { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the number of entries in the node.
|
/// Gets or sets the number of entries in the node.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ushort EntryCount { get; set; }
|
public ushort EntryCount { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the parent page identifier.
|
/// Gets or sets the parent page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint ParentPageId { get; set; }
|
public uint ParentPageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the next leaf page identifier.
|
/// Gets or sets the next leaf page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint NextLeafPageId { get; set; } // For leaf nodes only
|
public uint NextLeafPageId { get; set; } // For leaf nodes only
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the previous leaf page identifier.
|
/// Gets or sets the previous leaf page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PrevLeafPageId { get; set; } // For leaf nodes only (added for reverse scan)
|
public uint PrevLeafPageId { get; set; } // For leaf nodes only (added for reverse scan)
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes the header to a byte span.
|
/// Writes the header to a byte span.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span.</param>
|
/// <param name="destination">The destination span.</param>
|
||||||
public void WriteTo(Span<byte> destination)
|
public void WriteTo(Span<byte> destination)
|
||||||
{
|
{
|
||||||
if (destination.Length < 20)
|
if (destination.Length < 20)
|
||||||
throw new ArgumentException("Destination must be at least 20 bytes");
|
throw new ArgumentException("Destination must be at least 20 bytes");
|
||||||
|
|
||||||
BitConverter.TryWriteBytes(destination[0..4], PageId);
|
BitConverter.TryWriteBytes(destination[..4], PageId);
|
||||||
destination[4] = (byte)(IsLeaf ? 1 : 0);
|
destination[4] = (byte)(IsLeaf ? 1 : 0);
|
||||||
BitConverter.TryWriteBytes(destination[5..7], EntryCount);
|
BitConverter.TryWriteBytes(destination[5..7], EntryCount);
|
||||||
BitConverter.TryWriteBytes(destination[7..11], ParentPageId);
|
BitConverter.TryWriteBytes(destination[7..11], ParentPageId);
|
||||||
BitConverter.TryWriteBytes(destination[11..15], NextLeafPageId);
|
BitConverter.TryWriteBytes(destination[11..15], NextLeafPageId);
|
||||||
BitConverter.TryWriteBytes(destination[15..19], PrevLeafPageId);
|
BitConverter.TryWriteBytes(destination[15..19], PrevLeafPageId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a node header from a byte span.
|
/// Reads a node header from a byte span.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The source span.</param>
|
/// <param name="source">The source span.</param>
|
||||||
/// <returns>The parsed node header.</returns>
|
/// <returns>The parsed node header.</returns>
|
||||||
@@ -137,21 +136,18 @@ public struct BTreeNodeHeader
|
|||||||
{
|
{
|
||||||
if (source.Length < 20)
|
if (source.Length < 20)
|
||||||
throw new ArgumentException("Source must be at least 16 bytes");
|
throw new ArgumentException("Source must be at least 16 bytes");
|
||||||
|
|
||||||
var header = new BTreeNodeHeader
|
var header = new BTreeNodeHeader
|
||||||
{
|
{
|
||||||
PageId = BitConverter.ToUInt32(source[0..4]),
|
PageId = BitConverter.ToUInt32(source[..4]),
|
||||||
IsLeaf = source[4] != 0,
|
IsLeaf = source[4] != 0,
|
||||||
EntryCount = BitConverter.ToUInt16(source[5..7]),
|
EntryCount = BitConverter.ToUInt16(source[5..7]),
|
||||||
ParentPageId = BitConverter.ToUInt32(source[7..11]),
|
ParentPageId = BitConverter.ToUInt32(source[7..11]),
|
||||||
NextLeafPageId = BitConverter.ToUInt32(source[11..15])
|
NextLeafPageId = BitConverter.ToUInt32(source[11..15])
|
||||||
};
|
};
|
||||||
|
|
||||||
if (source.Length >= 20)
|
if (source.Length >= 20) header.PrevLeafPageId = BitConverter.ToUInt32(source[15..19]);
|
||||||
{
|
|
||||||
header.PrevLeafPageId = BitConverter.ToUInt32(source[15..19]);
|
return header;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return header;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,71 +1,26 @@
|
|||||||
using System;
|
|
||||||
using System.Linq.Expressions;
|
using System.Linq.Expressions;
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// High-level metadata and configuration for a custom index on a document collection.
|
/// High-level metadata and configuration for a custom index on a document collection.
|
||||||
/// Wraps low-level IndexOptions and provides strongly-typed expression-based key extraction.
|
/// Wraps low-level IndexOptions and provides strongly-typed expression-based key extraction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="T">Document type</typeparam>
|
/// <typeparam name="T">Document type</typeparam>
|
||||||
public sealed class CollectionIndexDefinition<T> where T : class
|
public sealed class CollectionIndexDefinition<T> where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Unique name for this index (auto-generated or user-specified)
|
/// Creates a new index definition
|
||||||
/// </summary>
|
|
||||||
public string Name { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Property paths that make up this index key.
|
|
||||||
/// Examples: ["Age"] for simple index, ["City", "Age"] for compound index
|
|
||||||
/// </summary>
|
|
||||||
public string[] PropertyPaths { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If true, enforces uniqueness constraint on the indexed values
|
|
||||||
/// </summary>
|
|
||||||
public bool IsUnique { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Type of index structure (from existing IndexType enum)
|
|
||||||
/// </summary>
|
|
||||||
public IndexType Type { get; }
|
|
||||||
|
|
||||||
/// <summary>Vector dimensions (only for Vector index)</summary>
|
|
||||||
public int Dimensions { get; }
|
|
||||||
|
|
||||||
/// <summary>Distance metric (only for Vector index)</summary>
|
|
||||||
public VectorMetric Metric { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compiled function to extract the index key from a document.
|
|
||||||
/// Compiled for maximum performance (10-100x faster than interpreting Expression).
|
|
||||||
/// </summary>
|
|
||||||
public Func<T, object> KeySelector { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Original expression for the key selector (for analysis and serialization)
|
|
||||||
/// </summary>
|
|
||||||
public Expression<Func<T, object>> KeySelectorExpression { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// If true, this is the primary key index (_id)
|
|
||||||
/// </summary>
|
|
||||||
public bool IsPrimary { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Creates a new index definition
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">Index name</param>
|
/// <param name="name">Index name</param>
|
||||||
/// <param name="propertyPaths">Property paths for the index</param>
|
/// <param name="propertyPaths">Property paths for the index</param>
|
||||||
/// <param name="keySelectorExpression">Expression to extract key from document</param>
|
/// <param name="keySelectorExpression">Expression to extract key from document</param>
|
||||||
/// <param name="isUnique">Enforce uniqueness</param>
|
/// <param name="isUnique">Enforce uniqueness</param>
|
||||||
/// <param name="type">Index structure type (BTree or Hash)</param>
|
/// <param name="type">Index structure type (BTree or Hash)</param>
|
||||||
/// <param name="isPrimary">Is this the primary key index</param>
|
/// <param name="isPrimary">Is this the primary key index</param>
|
||||||
/// <param name="dimensions">The vector dimensions for vector indexes.</param>
|
/// <param name="dimensions">The vector dimensions for vector indexes.</param>
|
||||||
/// <param name="metric">The distance metric for vector indexes.</param>
|
/// <param name="metric">The distance metric for vector indexes.</param>
|
||||||
public CollectionIndexDefinition(
|
public CollectionIndexDefinition(
|
||||||
string name,
|
string name,
|
||||||
string[] propertyPaths,
|
string[] propertyPaths,
|
||||||
Expression<Func<T, object>> keySelectorExpression,
|
Expression<Func<T, object>> keySelectorExpression,
|
||||||
@@ -76,11 +31,11 @@ public sealed class CollectionIndexDefinition<T> where T : class
|
|||||||
VectorMetric metric = VectorMetric.Cosine)
|
VectorMetric metric = VectorMetric.Cosine)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(name))
|
if (string.IsNullOrWhiteSpace(name))
|
||||||
throw new ArgumentException("Index name cannot be empty", nameof(name));
|
throw new ArgumentException("Index name cannot be empty", nameof(name));
|
||||||
|
|
||||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||||
throw new ArgumentException("Property paths cannot be empty", nameof(propertyPaths));
|
throw new ArgumentException("Property paths cannot be empty", nameof(propertyPaths));
|
||||||
|
|
||||||
Name = name;
|
Name = name;
|
||||||
PropertyPaths = propertyPaths;
|
PropertyPaths = propertyPaths;
|
||||||
KeySelectorExpression = keySelectorExpression ?? throw new ArgumentNullException(nameof(keySelectorExpression));
|
KeySelectorExpression = keySelectorExpression ?? throw new ArgumentNullException(nameof(keySelectorExpression));
|
||||||
@@ -90,10 +45,53 @@ public sealed class CollectionIndexDefinition<T> where T : class
|
|||||||
IsPrimary = isPrimary;
|
IsPrimary = isPrimary;
|
||||||
Dimensions = dimensions;
|
Dimensions = dimensions;
|
||||||
Metric = metric;
|
Metric = metric;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts this high-level definition to low-level IndexOptions for BTreeIndex
|
/// Unique name for this index (auto-generated or user-specified)
|
||||||
|
/// </summary>
|
||||||
|
public string Name { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Property paths that make up this index key.
|
||||||
|
/// Examples: ["Age"] for simple index, ["City", "Age"] for compound index
|
||||||
|
/// </summary>
|
||||||
|
public string[] PropertyPaths { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// If true, enforces uniqueness constraint on the indexed values
|
||||||
|
/// </summary>
|
||||||
|
public bool IsUnique { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Type of index structure (from existing IndexType enum)
|
||||||
|
/// </summary>
|
||||||
|
public IndexType Type { get; }
|
||||||
|
|
||||||
|
/// <summary>Vector dimensions (only for Vector index)</summary>
|
||||||
|
public int Dimensions { get; }
|
||||||
|
|
||||||
|
/// <summary>Distance metric (only for Vector index)</summary>
|
||||||
|
public VectorMetric Metric { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compiled function to extract the index key from a document.
|
||||||
|
/// Compiled for maximum performance (10-100x faster than interpreting Expression).
|
||||||
|
/// </summary>
|
||||||
|
public Func<T, object> KeySelector { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Original expression for the key selector (for analysis and serialization)
|
||||||
|
/// </summary>
|
||||||
|
public Expression<Func<T, object>> KeySelectorExpression { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// If true, this is the primary key index (_id)
|
||||||
|
/// </summary>
|
||||||
|
public bool IsPrimary { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Converts this high-level definition to low-level IndexOptions for BTreeIndex
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexOptions ToIndexOptions()
|
public IndexOptions ToIndexOptions()
|
||||||
{
|
{
|
||||||
@@ -105,98 +103,97 @@ public sealed class CollectionIndexDefinition<T> where T : class
|
|||||||
Dimensions = Dimensions,
|
Dimensions = Dimensions,
|
||||||
Metric = Metric
|
Metric = Metric
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Checks if this index can be used for a query on the specified property path
|
/// Checks if this index can be used for a query on the specified property path
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="propertyPath">The property path to validate.</param>
|
/// <param name="propertyPath">The property path to validate.</param>
|
||||||
public bool CanSupportQuery(string propertyPath)
|
public bool CanSupportQuery(string propertyPath)
|
||||||
{
|
{
|
||||||
// Simple index: exact match required
|
// Simple index: exact match required
|
||||||
if (PropertyPaths.Length == 1)
|
if (PropertyPaths.Length == 1)
|
||||||
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
||||||
|
|
||||||
// Compound index: can support if queried property is the first component
|
// Compound index: can support if queried property is the first component
|
||||||
// e.g., index on ["City", "Age"] can support query on "City" but not just "Age"
|
// e.g., index on ["City", "Age"] can support query on "City" but not just "Age"
|
||||||
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
return PropertyPaths[0].Equals(propertyPath, StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Checks if this index can support queries on multiple properties (compound queries)
|
/// Checks if this index can support queries on multiple properties (compound queries)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="propertyPaths">The ordered property paths to validate.</param>
|
/// <param name="propertyPaths">The ordered property paths to validate.</param>
|
||||||
public bool CanSupportCompoundQuery(string[] propertyPaths)
|
public bool CanSupportCompoundQuery(string[] propertyPaths)
|
||||||
{
|
{
|
||||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
// Check if queried paths are a prefix of this index
|
// Check if queried paths are a prefix of this index
|
||||||
// e.g., index on ["City", "Age", "Name"] can support ["City"] or ["City", "Age"]
|
// e.g., index on ["City", "Age", "Name"] can support ["City"] or ["City", "Age"]
|
||||||
if (propertyPaths.Length > PropertyPaths.Length)
|
if (propertyPaths.Length > PropertyPaths.Length)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
for (int i = 0; i < propertyPaths.Length; i++)
|
for (var i = 0; i < propertyPaths.Length; i++)
|
||||||
{
|
|
||||||
if (!PropertyPaths[i].Equals(propertyPaths[i], StringComparison.OrdinalIgnoreCase))
|
if (!PropertyPaths[i].Equals(propertyPaths[i], StringComparison.OrdinalIgnoreCase))
|
||||||
return false;
|
return false;
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
{
|
{
|
||||||
var uniqueStr = IsUnique ? "Unique" : "Non-Unique";
|
string uniqueStr = IsUnique ? "Unique" : "Non-Unique";
|
||||||
var paths = string.Join(", ", PropertyPaths);
|
string paths = string.Join(", ", PropertyPaths);
|
||||||
return $"{Name} ({uniqueStr} {Type} on [{paths}])";
|
return $"{Name} ({uniqueStr} {Type} on [{paths}])";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Information about an existing index (for querying index metadata)
|
/// Information about an existing index (for querying index metadata)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CollectionIndexInfo
|
public sealed class CollectionIndexInfo
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the index name.
|
/// Gets the index name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string Name { get; init; } = string.Empty;
|
public string Name { get; init; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the indexed property paths.
|
/// Gets the indexed property paths.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] PropertyPaths { get; init; } = Array.Empty<string>();
|
public string[] PropertyPaths { get; init; } = Array.Empty<string>();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether the index is unique.
|
/// Gets a value indicating whether the index is unique.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsUnique { get; init; }
|
public bool IsUnique { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the index type.
|
/// Gets the index type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexType Type { get; init; }
|
public IndexType Type { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether this index is the primary index.
|
/// Gets a value indicating whether this index is the primary index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsPrimary { get; init; }
|
public bool IsPrimary { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the estimated number of indexed documents.
|
/// Gets the estimated number of indexed documents.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long EstimatedDocumentCount { get; init; }
|
public long EstimatedDocumentCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the estimated storage size, in bytes.
|
/// Gets the estimated storage size, in bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long EstimatedSizeBytes { get; init; }
|
public long EstimatedSizeBytes { get; init; }
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
{
|
{
|
||||||
return $"{Name}: {string.Join(", ", PropertyPaths)} ({EstimatedDocumentCount} docs, {EstimatedSizeBytes:N0} bytes)";
|
return
|
||||||
|
$"{Name}: {string.Join(", ", PropertyPaths)} ({EstimatedDocumentCount} docs, {EstimatedSizeBytes:N0} bytes)";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,4 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq.Expressions;
|
using System.Linq.Expressions;
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
@@ -9,57 +6,94 @@ using ZB.MOM.WW.CBDD.Core.Transactions;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Manages a collection of secondary indexes on a document collection.
|
/// Manages a collection of secondary indexes on a document collection.
|
||||||
/// Handles index creation, deletion, automatic selection, and maintenance.
|
/// Handles index creation, deletion, automatic selection, and maintenance.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TId">Primary key type</typeparam>
|
/// <typeparam name="TId">Primary key type</typeparam>
|
||||||
/// <typeparam name="T">Document type</typeparam>
|
/// <typeparam name="T">Document type</typeparam>
|
||||||
public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
||||||
{
|
{
|
||||||
private readonly Dictionary<string, CollectionSecondaryIndex<TId, T>> _indexes;
|
private readonly string _collectionName;
|
||||||
private readonly IStorageEngine _storage;
|
private readonly Dictionary<string, CollectionSecondaryIndex<TId, T>> _indexes;
|
||||||
private readonly IDocumentMapper<TId, T> _mapper;
|
|
||||||
private readonly object _lock = new();
|
private readonly object _lock = new();
|
||||||
|
private readonly IDocumentMapper<TId, T> _mapper;
|
||||||
|
private readonly IStorageEngine _storage;
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
private readonly string _collectionName;
|
private CollectionMetadata _metadata;
|
||||||
private CollectionMetadata _metadata;
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Initializes a new instance of the <see cref="CollectionIndexManager{TId, T}" /> class.
|
||||||
/// Initializes a new instance of the <see cref="CollectionIndexManager{TId, T}"/> class.
|
/// </summary>
|
||||||
/// </summary>
|
/// <param name="storage">The storage engine used to persist index data and metadata.</param>
|
||||||
/// <param name="storage">The storage engine used to persist index data and metadata.</param>
|
/// <param name="mapper">The document mapper for the collection type.</param>
|
||||||
/// <param name="mapper">The document mapper for the collection type.</param>
|
/// <param name="collectionName">The optional collection name override.</param>
|
||||||
/// <param name="collectionName">The optional collection name override.</param>
|
public CollectionIndexManager(StorageEngine storage, IDocumentMapper<TId, T> mapper, string? collectionName = null)
|
||||||
public CollectionIndexManager(StorageEngine storage, IDocumentMapper<TId, T> mapper, string? collectionName = null)
|
: this((IStorageEngine)storage, mapper, collectionName)
|
||||||
: this((IStorageEngine)storage, mapper, collectionName)
|
{
|
||||||
{
|
}
|
||||||
}
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Initializes a new instance of the <see cref="CollectionIndexManager{TId, T}" /> class from the storage abstraction.
|
||||||
/// Initializes a new instance of the <see cref="CollectionIndexManager{TId, T}"/> class from the storage abstraction.
|
/// </summary>
|
||||||
/// </summary>
|
/// <param name="storage">The storage abstraction used to persist index state.</param>
|
||||||
/// <param name="storage">The storage abstraction used to persist index state.</param>
|
/// <param name="mapper">The document mapper for the collection.</param>
|
||||||
/// <param name="mapper">The document mapper for the collection.</param>
|
/// <param name="collectionName">An optional collection name override.</param>
|
||||||
/// <param name="collectionName">An optional collection name override.</param>
|
internal CollectionIndexManager(IStorageEngine storage, IDocumentMapper<TId, T> mapper,
|
||||||
internal CollectionIndexManager(IStorageEngine storage, IDocumentMapper<TId, T> mapper, string? collectionName = null)
|
string? collectionName = null)
|
||||||
{
|
{
|
||||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||||
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
||||||
_collectionName = collectionName ?? _mapper.CollectionName;
|
_collectionName = collectionName ?? _mapper.CollectionName;
|
||||||
_indexes = new Dictionary<string, CollectionSecondaryIndex<TId, T>>(StringComparer.OrdinalIgnoreCase);
|
_indexes = new Dictionary<string, CollectionSecondaryIndex<TId, T>>(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
// Load existing metadata via storage
|
// Load existing metadata via storage
|
||||||
_metadata = _storage.GetCollectionMetadata(_collectionName) ?? new CollectionMetadata { Name = _collectionName };
|
_metadata = _storage.GetCollectionMetadata(_collectionName) ??
|
||||||
|
new CollectionMetadata { Name = _collectionName };
|
||||||
// Initialize indexes from metadata
|
|
||||||
|
// Initialize indexes from metadata
|
||||||
foreach (var idxMeta in _metadata.Indexes)
|
foreach (var idxMeta in _metadata.Indexes)
|
||||||
{
|
{
|
||||||
var definition = RebuildDefinition(idxMeta.Name, idxMeta.PropertyPaths, idxMeta.IsUnique, idxMeta.Type, idxMeta.Dimensions, idxMeta.Metric);
|
var definition = RebuildDefinition(idxMeta.Name, idxMeta.PropertyPaths, idxMeta.IsUnique, idxMeta.Type,
|
||||||
|
idxMeta.Dimensions, idxMeta.Metric);
|
||||||
var index = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper, idxMeta.RootPageId);
|
var index = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper, idxMeta.RootPageId);
|
||||||
_indexes[idxMeta.Name] = index;
|
_indexes[idxMeta.Name] = index;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the root page identifier for the primary index.
|
||||||
|
/// </summary>
|
||||||
|
public uint PrimaryRootPageId => _metadata.PrimaryRootPageId;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Releases resources used by the index manager.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
return;
|
||||||
|
|
||||||
|
// No auto-save on dispose to avoid unnecessary I/O if no changes
|
||||||
|
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
foreach (var index in _indexes.Values)
|
||||||
|
try
|
||||||
|
{
|
||||||
|
index.Dispose();
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
/* Best effort */
|
||||||
|
}
|
||||||
|
|
||||||
|
_indexes.Clear();
|
||||||
|
_disposed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
}
|
||||||
|
|
||||||
private void UpdateMetadata()
|
private void UpdateMetadata()
|
||||||
{
|
{
|
||||||
_metadata.Indexes.Clear();
|
_metadata.Indexes.Clear();
|
||||||
@@ -80,7 +114,7 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a new secondary index
|
/// Creates a new secondary index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="definition">Index definition</param>
|
/// <param name="definition">Index definition</param>
|
||||||
/// <returns>The created secondary index</returns>
|
/// <returns>The created secondary index</returns>
|
||||||
@@ -100,9 +134,9 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
|
|
||||||
// Create secondary index
|
// Create secondary index
|
||||||
var secondaryIndex = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper);
|
var secondaryIndex = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper);
|
||||||
_indexes[definition.Name] = secondaryIndex;
|
_indexes[definition.Name] = secondaryIndex;
|
||||||
|
|
||||||
// Persist metadata
|
// Persist metadata
|
||||||
UpdateMetadata();
|
UpdateMetadata();
|
||||||
_storage.SaveCollectionMetadata(_metadata);
|
_storage.SaveCollectionMetadata(_metadata);
|
||||||
|
|
||||||
@@ -113,7 +147,7 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
// ... methods ...
|
// ... methods ...
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a simple index on a single property
|
/// Creates a simple index on a single property
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">Key type</typeparam>
|
/// <typeparam name="TKey">Key type</typeparam>
|
||||||
/// <param name="keySelector">Expression to extract key from document</param>
|
/// <param name="keySelector">Expression to extract key from document</param>
|
||||||
@@ -129,9 +163,9 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
throw new ArgumentNullException(nameof(keySelector));
|
throw new ArgumentNullException(nameof(keySelector));
|
||||||
|
|
||||||
// Extract property paths from expression
|
// Extract property paths from expression
|
||||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
string[] propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||||
|
|
||||||
// Generate name if not provided
|
// Generate name if not provided
|
||||||
name ??= GenerateIndexName(propertyPaths);
|
name ??= GenerateIndexName(propertyPaths);
|
||||||
|
|
||||||
// Convert expression to object-returning expression (required for definition)
|
// Convert expression to object-returning expression (required for definition)
|
||||||
@@ -149,52 +183,51 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
return CreateIndex(definition);
|
return CreateIndex(definition);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a vector index for a collection property.
|
/// Creates a vector index for a collection property.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">The selected key type.</typeparam>
|
/// <typeparam name="TKey">The selected key type.</typeparam>
|
||||||
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
||||||
/// <param name="dimensions">Vector dimensionality.</param>
|
/// <param name="dimensions">Vector dimensionality.</param>
|
||||||
/// <param name="metric">Distance metric used by the vector index.</param>
|
/// <param name="metric">Distance metric used by the vector index.</param>
|
||||||
/// <param name="name">Optional index name.</param>
|
/// <param name="name">Optional index name.</param>
|
||||||
/// <returns>The created or existing index.</returns>
|
/// <returns>The created or existing index.</returns>
|
||||||
public CollectionSecondaryIndex<TId, T> CreateVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector, int dimensions, VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
public CollectionSecondaryIndex<TId, T> CreateVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector,
|
||||||
{
|
int dimensions, VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
||||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
{
|
||||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
string[] propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||||
|
string indexName = name ?? GenerateIndexName(propertyPaths);
|
||||||
|
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
if (_indexes.TryGetValue(indexName, out var existing))
|
if (_indexes.TryGetValue(indexName, out var existing))
|
||||||
return existing;
|
return existing;
|
||||||
|
|
||||||
var body = keySelector.Body;
|
var body = keySelector.Body;
|
||||||
if (body.Type != typeof(object))
|
if (body.Type != typeof(object)) body = Expression.Convert(body, typeof(object));
|
||||||
{
|
|
||||||
body = Expression.Convert(body, typeof(object));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reuse the original parameter from keySelector to avoid invalid expression trees.
|
|
||||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
|
||||||
|
|
||||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric);
|
|
||||||
return CreateIndex(definition);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
// Reuse the original parameter from keySelector to avoid invalid expression trees.
|
||||||
/// Ensures that an index exists for the specified key selector.
|
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||||
/// </summary>
|
|
||||||
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector,
|
||||||
/// <param name="name">Optional index name.</param>
|
false, dimensions, metric);
|
||||||
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
return CreateIndex(definition);
|
||||||
/// <returns>The existing or newly created index.</returns>
|
}
|
||||||
public CollectionSecondaryIndex<TId, T> EnsureIndex(
|
}
|
||||||
Expression<Func<T, object>> keySelector,
|
|
||||||
string? name = null,
|
/// <summary>
|
||||||
bool unique = false)
|
/// Ensures that an index exists for the specified key selector.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="keySelector">Expression to extract the indexed field.</param>
|
||||||
|
/// <param name="name">Optional index name.</param>
|
||||||
|
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
||||||
|
/// <returns>The existing or newly created index.</returns>
|
||||||
|
public CollectionSecondaryIndex<TId, T> EnsureIndex(
|
||||||
|
Expression<Func<T, object>> keySelector,
|
||||||
|
string? name = null,
|
||||||
|
bool unique = false)
|
||||||
{
|
{
|
||||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
string[] propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||||
name ??= GenerateIndexName(propertyPaths);
|
name ??= GenerateIndexName(propertyPaths);
|
||||||
|
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
@@ -206,46 +239,43 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Ensures that an index exists for the specified untyped key selector.
|
/// Ensures that an index exists for the specified untyped key selector.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||||
/// <param name="name">Optional index name.</param>
|
/// <param name="name">Optional index name.</param>
|
||||||
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
/// <param name="unique">Whether the index enforces uniqueness.</param>
|
||||||
/// <returns>The existing or newly created index.</returns>
|
/// <returns>The existing or newly created index.</returns>
|
||||||
internal CollectionSecondaryIndex<TId, T> EnsureIndexUntyped(
|
internal CollectionSecondaryIndex<TId, T> EnsureIndexUntyped(
|
||||||
LambdaExpression keySelector,
|
LambdaExpression keySelector,
|
||||||
string? name = null,
|
string? name = null,
|
||||||
bool unique = false)
|
bool unique = false)
|
||||||
{
|
{
|
||||||
// Convert LambdaExpression to Expression<Func<T, object>> properly by sharing parameters
|
// Convert LambdaExpression to Expression<Func<T, object>> properly by sharing parameters
|
||||||
var body = keySelector.Body;
|
var body = keySelector.Body;
|
||||||
if (body.Type != typeof(object))
|
if (body.Type != typeof(object)) body = Expression.Convert(body, typeof(object));
|
||||||
{
|
|
||||||
body = Expression.Convert(body, typeof(object));
|
|
||||||
}
|
|
||||||
|
|
||||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||||
|
|
||||||
return EnsureIndex(lambda, name, unique);
|
return EnsureIndex(lambda, name, unique);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a vector index from an untyped key selector.
|
/// Creates a vector index from an untyped key selector.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||||
/// <param name="dimensions">Vector dimensionality.</param>
|
/// <param name="dimensions">Vector dimensionality.</param>
|
||||||
/// <param name="metric">Distance metric used by the vector index.</param>
|
/// <param name="metric">Distance metric used by the vector index.</param>
|
||||||
/// <param name="name">Optional index name.</param>
|
/// <param name="name">Optional index name.</param>
|
||||||
/// <returns>The created or existing index.</returns>
|
/// <returns>The created or existing index.</returns>
|
||||||
public CollectionSecondaryIndex<TId, T> CreateVectorIndexUntyped(
|
public CollectionSecondaryIndex<TId, T> CreateVectorIndexUntyped(
|
||||||
LambdaExpression keySelector,
|
LambdaExpression keySelector,
|
||||||
int dimensions,
|
int dimensions,
|
||||||
VectorMetric metric = VectorMetric.Cosine,
|
VectorMetric metric = VectorMetric.Cosine,
|
||||||
string? name = null)
|
string? name = null)
|
||||||
{
|
{
|
||||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
string[] propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
string indexName = name ?? GenerateIndexName(propertyPaths);
|
||||||
|
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
@@ -253,51 +283,47 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
return existing;
|
return existing;
|
||||||
|
|
||||||
var body = keySelector.Body;
|
var body = keySelector.Body;
|
||||||
if (body.Type != typeof(object))
|
if (body.Type != typeof(object)) body = Expression.Convert(body, typeof(object));
|
||||||
{
|
|
||||||
body = Expression.Convert(body, typeof(object));
|
|
||||||
}
|
|
||||||
|
|
||||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||||
|
|
||||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector, false, dimensions, metric);
|
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Vector,
|
||||||
return CreateIndex(definition);
|
false, dimensions, metric);
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Creates a spatial index from an untyped key selector.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
|
||||||
/// <param name="name">Optional index name.</param>
|
|
||||||
/// <returns>The created or existing index.</returns>
|
|
||||||
public CollectionSecondaryIndex<TId, T> CreateSpatialIndexUntyped(
|
|
||||||
LambdaExpression keySelector,
|
|
||||||
string? name = null)
|
|
||||||
{
|
|
||||||
var propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
|
||||||
var indexName = name ?? GenerateIndexName(propertyPaths);
|
|
||||||
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
if (_indexes.TryGetValue(indexName, out var existing))
|
|
||||||
return existing;
|
|
||||||
|
|
||||||
var body = keySelector.Body;
|
|
||||||
if (body.Type != typeof(object))
|
|
||||||
{
|
|
||||||
body = Expression.Convert(body, typeof(object));
|
|
||||||
}
|
|
||||||
|
|
||||||
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
|
||||||
|
|
||||||
var definition = new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Spatial);
|
|
||||||
return CreateIndex(definition);
|
return CreateIndex(definition);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Drops an existing index by name
|
/// Creates a spatial index from an untyped key selector.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="keySelector">Untyped expression that selects indexed fields.</param>
|
||||||
|
/// <param name="name">Optional index name.</param>
|
||||||
|
/// <returns>The created or existing index.</returns>
|
||||||
|
public CollectionSecondaryIndex<TId, T> CreateSpatialIndexUntyped(
|
||||||
|
LambdaExpression keySelector,
|
||||||
|
string? name = null)
|
||||||
|
{
|
||||||
|
string[] propertyPaths = ExpressionAnalyzer.ExtractPropertyPaths(keySelector);
|
||||||
|
string indexName = name ?? GenerateIndexName(propertyPaths);
|
||||||
|
|
||||||
|
lock (_lock)
|
||||||
|
{
|
||||||
|
if (_indexes.TryGetValue(indexName, out var existing))
|
||||||
|
return existing;
|
||||||
|
|
||||||
|
var body = keySelector.Body;
|
||||||
|
if (body.Type != typeof(object)) body = Expression.Convert(body, typeof(object));
|
||||||
|
|
||||||
|
var lambda = Expression.Lambda<Func<T, object>>(body, keySelector.Parameters);
|
||||||
|
|
||||||
|
var definition =
|
||||||
|
new CollectionIndexDefinition<T>(indexName, propertyPaths, lambda, false, IndexType.Spatial);
|
||||||
|
return CreateIndex(definition);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Drops an existing index by name
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">Index name</param>
|
/// <param name="name">Index name</param>
|
||||||
/// <returns>True if index was found and dropped, false otherwise</returns>
|
/// <returns>True if index was found and dropped, false otherwise</returns>
|
||||||
@@ -311,10 +337,10 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
if (_indexes.TryGetValue(name, out var index))
|
if (_indexes.TryGetValue(name, out var index))
|
||||||
{
|
{
|
||||||
index.Dispose();
|
index.Dispose();
|
||||||
_indexes.Remove(name);
|
_indexes.Remove(name);
|
||||||
|
|
||||||
// TODO: Free pages used by index in PageFile
|
// TODO: Free pages used by index in PageFile
|
||||||
|
|
||||||
SaveMetadata(); // Save metadata after dropping index
|
SaveMetadata(); // Save metadata after dropping index
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -323,11 +349,11 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets an index by name
|
/// Gets an index by name
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The index name.</param>
|
/// <param name="name">The index name.</param>
|
||||||
public CollectionSecondaryIndex<TId, T>? GetIndex(string name)
|
public CollectionSecondaryIndex<TId, T>? GetIndex(string name)
|
||||||
{
|
{
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
@@ -336,7 +362,7 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets all indexes
|
/// Gets all indexes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IEnumerable<CollectionSecondaryIndex<TId, T>> GetAllIndexes()
|
public IEnumerable<CollectionSecondaryIndex<TId, T>> GetAllIndexes()
|
||||||
{
|
{
|
||||||
@@ -347,7 +373,7 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets information about all indexes
|
/// Gets information about all indexes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IEnumerable<CollectionIndexInfo> GetIndexInfo()
|
public IEnumerable<CollectionIndexInfo> GetIndexInfo()
|
||||||
{
|
{
|
||||||
@@ -358,8 +384,8 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Finds the best index to use for a query on the specified property.
|
/// Finds the best index to use for a query on the specified property.
|
||||||
/// Returns null if no suitable index found (requires full scan).
|
/// Returns null if no suitable index found (requires full scan).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="propertyPath">Property path being queried</param>
|
/// <param name="propertyPath">Property path being queried</param>
|
||||||
/// <returns>Best index for the query, or null if none suitable</returns>
|
/// <returns>Best index for the query, or null if none suitable</returns>
|
||||||
@@ -386,11 +412,11 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Finds the best index for a compound query on multiple properties
|
/// Finds the best index for a compound query on multiple properties
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="propertyPaths">The ordered list of queried property paths.</param>
|
/// <param name="propertyPaths">The ordered list of queried property paths.</param>
|
||||||
public CollectionSecondaryIndex<TId, T>? FindBestCompoundIndex(string[] propertyPaths)
|
public CollectionSecondaryIndex<TId, T>? FindBestCompoundIndex(string[] propertyPaths)
|
||||||
{
|
{
|
||||||
if (propertyPaths == null || propertyPaths.Length == 0)
|
if (propertyPaths == null || propertyPaths.Length == 0)
|
||||||
return null;
|
return null;
|
||||||
@@ -413,7 +439,7 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a document into all indexes
|
/// Inserts a document into all indexes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="document">Document to insert</param>
|
/// <param name="document">Document to insert</param>
|
||||||
/// <param name="location">Physical location of the document</param>
|
/// <param name="location">Physical location of the document</param>
|
||||||
@@ -425,22 +451,20 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
|
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
foreach (var index in _indexes.Values)
|
foreach (var index in _indexes.Values) index.Insert(document, location, transaction);
|
||||||
{
|
|
||||||
index.Insert(document, location, transaction);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Updates a document in all indexes
|
/// Updates a document in all indexes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="oldDocument">Old version of document</param>
|
/// <param name="oldDocument">Old version of document</param>
|
||||||
/// <param name="newDocument">New version of document</param>
|
/// <param name="newDocument">New version of document</param>
|
||||||
/// <param name="oldLocation">Physical location of old document</param>
|
/// <param name="oldLocation">Physical location of old document</param>
|
||||||
/// <param name="newLocation">Physical location of new document</param>
|
/// <param name="newLocation">Physical location of new document</param>
|
||||||
/// <param name="transaction">Transaction context</param>
|
/// <param name="transaction">Transaction context</param>
|
||||||
public void UpdateInAll(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
|
public void UpdateInAll(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation,
|
||||||
|
ITransaction transaction)
|
||||||
{
|
{
|
||||||
if (oldDocument == null)
|
if (oldDocument == null)
|
||||||
throw new ArgumentNullException(nameof(oldDocument));
|
throw new ArgumentNullException(nameof(oldDocument));
|
||||||
@@ -450,14 +474,12 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
foreach (var index in _indexes.Values)
|
foreach (var index in _indexes.Values)
|
||||||
{
|
|
||||||
index.Update(oldDocument, newDocument, oldLocation, newLocation, transaction);
|
index.Update(oldDocument, newDocument, oldLocation, newLocation, transaction);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deletes a document from all indexes
|
/// Deletes a document from all indexes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="document">Document to delete</param>
|
/// <param name="document">Document to delete</param>
|
||||||
/// <param name="location">Physical location of the document</param>
|
/// <param name="location">Physical location of the document</param>
|
||||||
@@ -469,83 +491,78 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
|
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
foreach (var index in _indexes.Values)
|
foreach (var index in _indexes.Values) index.Delete(document, location, transaction);
|
||||||
{
|
|
||||||
index.Delete(document, location, transaction);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Generates an index name from property paths
|
/// Generates an index name from property paths
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static string GenerateIndexName(string[] propertyPaths)
|
private static string GenerateIndexName(string[] propertyPaths)
|
||||||
{
|
{
|
||||||
return $"idx_{string.Join("_", propertyPaths)}";
|
return $"idx_{string.Join("_", propertyPaths)}";
|
||||||
}
|
}
|
||||||
|
|
||||||
private CollectionIndexDefinition<T> RebuildDefinition(string name, string[] paths, bool isUnique, IndexType type, int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
private CollectionIndexDefinition<T> RebuildDefinition(string name, string[] paths, bool isUnique, IndexType type,
|
||||||
|
int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
||||||
{
|
{
|
||||||
var param = Expression.Parameter(typeof(T), "u");
|
var param = Expression.Parameter(typeof(T), "u");
|
||||||
Expression body;
|
Expression body;
|
||||||
|
|
||||||
if (paths.Length == 1)
|
if (paths.Length == 1)
|
||||||
{
|
|
||||||
body = Expression.PropertyOrField(param, paths[0]);
|
body = Expression.PropertyOrField(param, paths[0]);
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
body = Expression.NewArrayInit(typeof(object),
|
||||||
body = Expression.NewArrayInit(typeof(object),
|
|
||||||
paths.Select(p => Expression.Convert(Expression.PropertyOrField(param, p), typeof(object))));
|
paths.Select(p => Expression.Convert(Expression.PropertyOrField(param, p), typeof(object))));
|
||||||
}
|
|
||||||
|
|
||||||
var objectBody = Expression.Convert(body, typeof(object));
|
var objectBody = Expression.Convert(body, typeof(object));
|
||||||
var lambda = Expression.Lambda<Func<T, object>>(objectBody, param);
|
var lambda = Expression.Lambda<Func<T, object>>(objectBody, param);
|
||||||
|
|
||||||
return new CollectionIndexDefinition<T>(name, paths, lambda, isUnique, type, false, dimensions, metric);
|
return new CollectionIndexDefinition<T>(name, paths, lambda, isUnique, type, false, dimensions, metric);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the root page identifier for the primary index.
|
/// Rebinds cached metadata and index instances from persisted metadata.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PrimaryRootPageId => _metadata.PrimaryRootPageId;
|
/// <param name="metadata">The collection metadata used to rebuild index state.</param>
|
||||||
|
internal void RebindFromMetadata(CollectionMetadata metadata)
|
||||||
/// <summary>
|
{
|
||||||
/// Rebinds cached metadata and index instances from persisted metadata.
|
if (metadata == null)
|
||||||
/// </summary>
|
throw new ArgumentNullException(nameof(metadata));
|
||||||
/// <param name="metadata">The collection metadata used to rebuild index state.</param>
|
|
||||||
internal void RebindFromMetadata(CollectionMetadata metadata)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
if (metadata == null)
|
if (_disposed)
|
||||||
throw new ArgumentNullException(nameof(metadata));
|
throw new ObjectDisposedException(nameof(CollectionIndexManager<TId, T>));
|
||||||
|
|
||||||
lock (_lock)
|
foreach (var index in _indexes.Values)
|
||||||
{
|
try
|
||||||
if (_disposed)
|
{
|
||||||
throw new ObjectDisposedException(nameof(CollectionIndexManager<TId, T>));
|
index.Dispose();
|
||||||
|
}
|
||||||
foreach (var index in _indexes.Values)
|
catch
|
||||||
{
|
{
|
||||||
try { index.Dispose(); } catch { /* Best effort */ }
|
/* Best effort */
|
||||||
}
|
}
|
||||||
|
|
||||||
_indexes.Clear();
|
_indexes.Clear();
|
||||||
_metadata = metadata;
|
_metadata = metadata;
|
||||||
|
|
||||||
foreach (var idxMeta in _metadata.Indexes)
|
foreach (var idxMeta in _metadata.Indexes)
|
||||||
{
|
{
|
||||||
var definition = RebuildDefinition(idxMeta.Name, idxMeta.PropertyPaths, idxMeta.IsUnique, idxMeta.Type, idxMeta.Dimensions, idxMeta.Metric);
|
var definition = RebuildDefinition(idxMeta.Name, idxMeta.PropertyPaths, idxMeta.IsUnique, idxMeta.Type,
|
||||||
var index = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper, idxMeta.RootPageId);
|
idxMeta.Dimensions, idxMeta.Metric);
|
||||||
_indexes[idxMeta.Name] = index;
|
var index = new CollectionSecondaryIndex<TId, T>(definition, _storage, _mapper, idxMeta.RootPageId);
|
||||||
}
|
_indexes[idxMeta.Name] = index;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Sets the root page identifier for the primary index.
|
/// <summary>
|
||||||
/// </summary>
|
/// Sets the root page identifier for the primary index.
|
||||||
/// <param name="pageId">The root page identifier.</param>
|
/// </summary>
|
||||||
public void SetPrimaryRootPageId(uint pageId)
|
/// <param name="pageId">The root page identifier.</param>
|
||||||
|
public void SetPrimaryRootPageId(uint pageId)
|
||||||
{
|
{
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
@@ -557,88 +574,62 @@ public sealed class CollectionIndexManager<TId, T> : IDisposable where T : class
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current collection metadata.
|
/// Gets the current collection metadata.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The collection metadata.</returns>
|
/// <returns>The collection metadata.</returns>
|
||||||
public CollectionMetadata GetMetadata() => _metadata;
|
public CollectionMetadata GetMetadata()
|
||||||
|
|
||||||
private void SaveMetadata()
|
|
||||||
{
|
|
||||||
UpdateMetadata();
|
|
||||||
_storage.SaveCollectionMetadata(_metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Releases resources used by the index manager.
|
|
||||||
/// </summary>
|
|
||||||
public void Dispose()
|
|
||||||
{
|
{
|
||||||
if (_disposed)
|
return _metadata;
|
||||||
return;
|
}
|
||||||
|
|
||||||
// No auto-save on dispose to avoid unnecessary I/O if no changes
|
|
||||||
|
|
||||||
lock (_lock)
|
|
||||||
{
|
|
||||||
foreach (var index in _indexes.Values)
|
|
||||||
{
|
|
||||||
try { index.Dispose(); } catch { /* Best effort */ }
|
|
||||||
}
|
|
||||||
|
|
||||||
_indexes.Clear();
|
|
||||||
_disposed = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
GC.SuppressFinalize(this);
|
private void SaveMetadata()
|
||||||
|
{
|
||||||
|
UpdateMetadata();
|
||||||
|
_storage.SaveCollectionMetadata(_metadata);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Helper class to analyze LINQ expressions and extract property paths
|
/// Helper class to analyze LINQ expressions and extract property paths
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static class ExpressionAnalyzer
|
public static class ExpressionAnalyzer
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Extracts property paths from a lambda expression.
|
/// Extracts property paths from a lambda expression.
|
||||||
/// Supports simple property access (p => p.Age) and anonymous types (p => new { p.City, p.Age }).
|
/// Supports simple property access (p => p.Age) and anonymous types (p => new { p.City, p.Age }).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="expression">The lambda expression to analyze.</param>
|
/// <param name="expression">The lambda expression to analyze.</param>
|
||||||
public static string[] ExtractPropertyPaths(LambdaExpression expression)
|
public static string[] ExtractPropertyPaths(LambdaExpression expression)
|
||||||
{
|
{
|
||||||
if (expression.Body is MemberExpression memberExpr)
|
if (expression.Body is MemberExpression memberExpr)
|
||||||
{
|
|
||||||
// Simple property: p => p.Age
|
// Simple property: p => p.Age
|
||||||
return new[] { memberExpr.Member.Name };
|
return new[] { memberExpr.Member.Name };
|
||||||
}
|
|
||||||
else if (expression.Body is NewExpression newExpr)
|
if (expression.Body is NewExpression newExpr)
|
||||||
{
|
|
||||||
// Compound key via anonymous type: p => new { p.City, p.Age }
|
// Compound key via anonymous type: p => new { p.City, p.Age }
|
||||||
return newExpr.Arguments
|
return newExpr.Arguments
|
||||||
.OfType<MemberExpression>()
|
.OfType<MemberExpression>()
|
||||||
.Select(m => m.Member.Name)
|
.Select(m => m.Member.Name)
|
||||||
.ToArray();
|
.ToArray();
|
||||||
}
|
|
||||||
else if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } unaryExpr)
|
if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } unaryExpr)
|
||||||
{
|
{
|
||||||
// Handle Convert(Member) or Convert(New)
|
// Handle Convert(Member) or Convert(New)
|
||||||
if (unaryExpr.Operand is MemberExpression innerMember)
|
if (unaryExpr.Operand is MemberExpression innerMember)
|
||||||
{
|
|
||||||
// Wrapped property: p => (object)p.Age
|
// Wrapped property: p => (object)p.Age
|
||||||
return new[] { innerMember.Member.Name };
|
return new[] { innerMember.Member.Name };
|
||||||
}
|
|
||||||
else if (unaryExpr.Operand is NewExpression innerNew)
|
if (unaryExpr.Operand is NewExpression innerNew)
|
||||||
{
|
// Wrapped anonymous type: p => (object)new { p.City, p.Age }
|
||||||
// Wrapped anonymous type: p => (object)new { p.City, p.Age }
|
return innerNew.Arguments
|
||||||
return innerNew.Arguments
|
.OfType<MemberExpression>()
|
||||||
.OfType<MemberExpression>()
|
.Select(m => m.Member.Name)
|
||||||
.Select(m => m.Member.Name)
|
.ToArray();
|
||||||
.ToArray();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new ArgumentException(
|
throw new ArgumentException(
|
||||||
"Expression must be a property accessor (p => p.Property) or anonymous type (p => new { p.Prop1, p.Prop2 })",
|
"Expression must be a property accessor (p => p.Property) or anonymous type (p => new { p.Prop1, p.Prop2 })",
|
||||||
nameof(expression));
|
nameof(expression));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,101 +1,112 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
|
||||||
using System;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a secondary (non-primary) index on a document collection.
|
/// Represents a secondary (non-primary) index on a document collection.
|
||||||
/// Provides a high-level, strongly-typed wrapper around the low-level BTreeIndex.
|
/// Provides a high-level, strongly-typed wrapper around the low-level BTreeIndex.
|
||||||
/// Handles automatic key extraction from documents using compiled expressions.
|
/// Handles automatic key extraction from documents using compiled expressions.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TId">Primary key type</typeparam>
|
/// <typeparam name="TId">Primary key type</typeparam>
|
||||||
/// <typeparam name="T">Document type</typeparam>
|
/// <typeparam name="T">Document type</typeparam>
|
||||||
public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : class
|
public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : class
|
||||||
{
|
{
|
||||||
private readonly CollectionIndexDefinition<T> _definition;
|
|
||||||
private readonly BTreeIndex? _btreeIndex;
|
|
||||||
private readonly VectorSearchIndex? _vectorIndex;
|
|
||||||
private readonly RTreeIndex? _spatialIndex;
|
|
||||||
private readonly IDocumentMapper<TId, T> _mapper;
|
private readonly IDocumentMapper<TId, T> _mapper;
|
||||||
|
private readonly RTreeIndex? _spatialIndex;
|
||||||
|
private readonly VectorSearchIndex? _vectorIndex;
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the index definition
|
/// Initializes a new instance of the <see cref="CollectionSecondaryIndex{TId, T}" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CollectionIndexDefinition<T> Definition => _definition;
|
/// <param name="definition">The index definition.</param>
|
||||||
|
/// <param name="storage">The storage engine.</param>
|
||||||
/// <summary>
|
/// <param name="mapper">The document mapper.</param>
|
||||||
/// Gets the underlying BTree index (for advanced scenarios)
|
/// <param name="rootPageId">The existing root page ID, or <c>0</c> to create a new one.</param>
|
||||||
/// </summary>
|
public CollectionSecondaryIndex(
|
||||||
public BTreeIndex? BTreeIndex => _btreeIndex;
|
CollectionIndexDefinition<T> definition,
|
||||||
|
StorageEngine storage,
|
||||||
/// <summary>
|
IDocumentMapper<TId, T> mapper,
|
||||||
/// Gets the root page identifier for the underlying index structure.
|
uint rootPageId = 0)
|
||||||
/// </summary>
|
: this(definition, (IStorageEngine)storage, mapper, rootPageId)
|
||||||
public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new instance of the <see cref="CollectionSecondaryIndex{TId, T}"/> class.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="definition">The index definition.</param>
|
|
||||||
/// <param name="storage">The storage engine.</param>
|
|
||||||
/// <param name="mapper">The document mapper.</param>
|
|
||||||
/// <param name="rootPageId">The existing root page ID, or <c>0</c> to create a new one.</param>
|
|
||||||
public CollectionSecondaryIndex(
|
|
||||||
CollectionIndexDefinition<T> definition,
|
|
||||||
StorageEngine storage,
|
|
||||||
IDocumentMapper<TId, T> mapper,
|
|
||||||
uint rootPageId = 0)
|
|
||||||
: this(definition, (IStorageEngine)storage, mapper, rootPageId)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new instance of the <see cref="CollectionSecondaryIndex{TId, T}"/> class from index storage abstractions.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="definition">The index definition.</param>
|
|
||||||
/// <param name="storage">The index storage abstraction.</param>
|
|
||||||
/// <param name="mapper">The document mapper.</param>
|
|
||||||
/// <param name="rootPageId">The existing root page identifier, if any.</param>
|
|
||||||
internal CollectionSecondaryIndex(
|
|
||||||
CollectionIndexDefinition<T> definition,
|
|
||||||
IIndexStorage storage,
|
|
||||||
IDocumentMapper<TId, T> mapper,
|
|
||||||
uint rootPageId = 0)
|
|
||||||
{
|
{
|
||||||
_definition = definition ?? throw new ArgumentNullException(nameof(definition));
|
}
|
||||||
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
|
||||||
|
/// <summary>
|
||||||
var indexOptions = definition.ToIndexOptions();
|
/// Initializes a new instance of the <see cref="CollectionSecondaryIndex{TId, T}" /> class from index storage
|
||||||
|
/// abstractions.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="definition">The index definition.</param>
|
||||||
|
/// <param name="storage">The index storage abstraction.</param>
|
||||||
|
/// <param name="mapper">The document mapper.</param>
|
||||||
|
/// <param name="rootPageId">The existing root page identifier, if any.</param>
|
||||||
|
internal CollectionSecondaryIndex(
|
||||||
|
CollectionIndexDefinition<T> definition,
|
||||||
|
IIndexStorage storage,
|
||||||
|
IDocumentMapper<TId, T> mapper,
|
||||||
|
uint rootPageId = 0)
|
||||||
|
{
|
||||||
|
Definition = definition ?? throw new ArgumentNullException(nameof(definition));
|
||||||
|
_mapper = mapper ?? throw new ArgumentNullException(nameof(mapper));
|
||||||
|
|
||||||
|
var indexOptions = definition.ToIndexOptions();
|
||||||
|
|
||||||
if (indexOptions.Type == IndexType.Vector)
|
if (indexOptions.Type == IndexType.Vector)
|
||||||
{
|
{
|
||||||
_vectorIndex = new VectorSearchIndex(storage, indexOptions, rootPageId);
|
_vectorIndex = new VectorSearchIndex(storage, indexOptions, rootPageId);
|
||||||
_btreeIndex = null;
|
BTreeIndex = null;
|
||||||
_spatialIndex = null;
|
_spatialIndex = null;
|
||||||
}
|
}
|
||||||
else if (indexOptions.Type == IndexType.Spatial)
|
else if (indexOptions.Type == IndexType.Spatial)
|
||||||
{
|
{
|
||||||
_spatialIndex = new RTreeIndex(storage, indexOptions, rootPageId);
|
_spatialIndex = new RTreeIndex(storage, indexOptions, rootPageId);
|
||||||
_btreeIndex = null;
|
BTreeIndex = null;
|
||||||
_vectorIndex = null;
|
_vectorIndex = null;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_btreeIndex = new BTreeIndex(storage, indexOptions, rootPageId);
|
BTreeIndex = new BTreeIndex(storage, indexOptions, rootPageId);
|
||||||
_vectorIndex = null;
|
_vectorIndex = null;
|
||||||
_spatialIndex = null;
|
_spatialIndex = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a document into this index
|
/// Gets the index definition
|
||||||
|
/// </summary>
|
||||||
|
public CollectionIndexDefinition<T> Definition { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the underlying BTree index (for advanced scenarios)
|
||||||
|
/// </summary>
|
||||||
|
public BTreeIndex? BTreeIndex { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the root page identifier for the underlying index structure.
|
||||||
|
/// </summary>
|
||||||
|
public uint RootPageId => BTreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Releases resources used by this index wrapper.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
if (_disposed)
|
||||||
|
return;
|
||||||
|
|
||||||
|
// BTreeIndex doesn't currently implement IDisposable
|
||||||
|
// Future: may need to flush buffers, close resources
|
||||||
|
|
||||||
|
_disposed = true;
|
||||||
|
GC.SuppressFinalize(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Inserts a document into this index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="document">Document to index</param>
|
/// <param name="document">Document to index</param>
|
||||||
/// <param name="location">Physical location of the document</param>
|
/// <param name="location">Physical location of the document</param>
|
||||||
@@ -103,91 +114,84 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
public void Insert(T document, DocumentLocation location, ITransaction transaction)
|
public void Insert(T document, DocumentLocation location, ITransaction transaction)
|
||||||
{
|
{
|
||||||
if (document == null)
|
if (document == null)
|
||||||
throw new ArgumentNullException(nameof(document));
|
throw new ArgumentNullException(nameof(document));
|
||||||
|
|
||||||
// Extract key using compiled selector (fast!)
|
// Extract key using compiled selector (fast!)
|
||||||
var keyValue = _definition.KeySelector(document);
|
object? keyValue = Definition.KeySelector(document);
|
||||||
if (keyValue == null)
|
if (keyValue == null)
|
||||||
return; // Skip null keys
|
return; // Skip null keys
|
||||||
|
|
||||||
if (_vectorIndex != null)
|
if (_vectorIndex != null)
|
||||||
{
|
{
|
||||||
// Vector Index Support
|
// Vector Index Support
|
||||||
if (keyValue is float[] singleVector)
|
if (keyValue is float[] singleVector)
|
||||||
{
|
|
||||||
_vectorIndex.Insert(singleVector, location, transaction);
|
_vectorIndex.Insert(singleVector, location, transaction);
|
||||||
}
|
|
||||||
else if (keyValue is IEnumerable<float[]> vectors)
|
else if (keyValue is IEnumerable<float[]> vectors)
|
||||||
{
|
foreach (float[] v in vectors)
|
||||||
foreach (var v in vectors)
|
|
||||||
{
|
|
||||||
_vectorIndex.Insert(v, location, transaction);
|
_vectorIndex.Insert(v, location, transaction);
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (_spatialIndex != null)
|
else if (_spatialIndex != null)
|
||||||
{
|
{
|
||||||
// Geospatial Index Support
|
// Geospatial Index Support
|
||||||
if (keyValue is ValueTuple<double, double> t)
|
if (keyValue is ValueTuple<double, double> t)
|
||||||
{
|
|
||||||
_spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
|
_spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (_btreeIndex != null)
|
else if (BTreeIndex != null)
|
||||||
{
|
{
|
||||||
// BTree Index logic
|
// BTree Index logic
|
||||||
var userKey = ConvertToIndexKey(keyValue);
|
var userKey = ConvertToIndexKey(keyValue);
|
||||||
var documentId = _mapper.GetId(document);
|
var documentId = _mapper.GetId(document);
|
||||||
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
||||||
_btreeIndex.Insert(compositeKey, location, transaction?.TransactionId);
|
BTreeIndex.Insert(compositeKey, location, transaction?.TransactionId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Updates a document in this index (delete old, insert new).
|
/// Updates a document in this index (delete old, insert new).
|
||||||
/// Only updates if the indexed key has changed.
|
/// Only updates if the indexed key has changed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="oldDocument">Old version of document</param>
|
/// <param name="oldDocument">Old version of document</param>
|
||||||
/// <param name="newDocument">New version of document</param>
|
/// <param name="newDocument">New version of document</param>
|
||||||
/// <param name="oldLocation">Physical location of old document</param>
|
/// <param name="oldLocation">Physical location of old document</param>
|
||||||
/// <param name="newLocation">Physical location of new document</param>
|
/// <param name="newLocation">Physical location of new document</param>
|
||||||
/// <param name="transaction">Optional transaction</param>
|
/// <param name="transaction">Optional transaction</param>
|
||||||
public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction)
|
public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation,
|
||||||
|
ITransaction transaction)
|
||||||
{
|
{
|
||||||
if (oldDocument == null)
|
if (oldDocument == null)
|
||||||
throw new ArgumentNullException(nameof(oldDocument));
|
throw new ArgumentNullException(nameof(oldDocument));
|
||||||
if (newDocument == null)
|
if (newDocument == null)
|
||||||
throw new ArgumentNullException(nameof(newDocument));
|
throw new ArgumentNullException(nameof(newDocument));
|
||||||
|
|
||||||
// Extract keys from both versions
|
// Extract keys from both versions
|
||||||
var oldKey = _definition.KeySelector(oldDocument);
|
object? oldKey = Definition.KeySelector(oldDocument);
|
||||||
var newKey = _definition.KeySelector(newDocument);
|
object? newKey = Definition.KeySelector(newDocument);
|
||||||
|
|
||||||
// If keys are the same, no index update needed (optimization)
|
// If keys are the same, no index update needed (optimization)
|
||||||
if (Equals(oldKey, newKey))
|
if (Equals(oldKey, newKey))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
var documentId = _mapper.GetId(oldDocument);
|
var documentId = _mapper.GetId(oldDocument);
|
||||||
|
|
||||||
// Delete old entry if it had a key
|
// Delete old entry if it had a key
|
||||||
if (oldKey != null)
|
if (oldKey != null)
|
||||||
{
|
{
|
||||||
var oldUserKey = ConvertToIndexKey(oldKey);
|
var oldUserKey = ConvertToIndexKey(oldKey);
|
||||||
var oldCompositeKey = CreateCompositeKey(oldUserKey, _mapper.ToIndexKey(documentId));
|
var oldCompositeKey = CreateCompositeKey(oldUserKey, _mapper.ToIndexKey(documentId));
|
||||||
_btreeIndex?.Delete(oldCompositeKey, oldLocation, transaction?.TransactionId);
|
BTreeIndex?.Delete(oldCompositeKey, oldLocation, transaction?.TransactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Insert new entry if it has a key
|
// Insert new entry if it has a key
|
||||||
if (newKey != null)
|
if (newKey != null)
|
||||||
{
|
{
|
||||||
var newUserKey = ConvertToIndexKey(newKey);
|
var newUserKey = ConvertToIndexKey(newKey);
|
||||||
var newCompositeKey = CreateCompositeKey(newUserKey, _mapper.ToIndexKey(documentId));
|
var newCompositeKey = CreateCompositeKey(newUserKey, _mapper.ToIndexKey(documentId));
|
||||||
_btreeIndex?.Insert(newCompositeKey, newLocation, transaction?.TransactionId);
|
BTreeIndex?.Insert(newCompositeKey, newLocation, transaction?.TransactionId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deletes a document from this index
|
/// Deletes a document from this index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="document">Document to remove from index</param>
|
/// <param name="document">Document to remove from index</param>
|
||||||
/// <param name="location">Physical location of the document</param>
|
/// <param name="location">Physical location of the document</param>
|
||||||
@@ -195,23 +199,23 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
public void Delete(T document, DocumentLocation location, ITransaction transaction)
|
public void Delete(T document, DocumentLocation location, ITransaction transaction)
|
||||||
{
|
{
|
||||||
if (document == null)
|
if (document == null)
|
||||||
throw new ArgumentNullException(nameof(document));
|
throw new ArgumentNullException(nameof(document));
|
||||||
|
|
||||||
// Extract key
|
// Extract key
|
||||||
var keyValue = _definition.KeySelector(document);
|
object? keyValue = Definition.KeySelector(document);
|
||||||
if (keyValue == null)
|
if (keyValue == null)
|
||||||
return; // Nothing to delete
|
return; // Nothing to delete
|
||||||
|
|
||||||
var userKey = ConvertToIndexKey(keyValue);
|
var userKey = ConvertToIndexKey(keyValue);
|
||||||
var documentId = _mapper.GetId(document);
|
var documentId = _mapper.GetId(document);
|
||||||
|
|
||||||
// Create composite key and delete
|
// Create composite key and delete
|
||||||
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId));
|
||||||
_btreeIndex?.Delete(compositeKey, location, transaction?.TransactionId);
|
BTreeIndex?.Delete(compositeKey, location, transaction?.TransactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Seeks a single document by exact key match (O(log n))
|
/// Seeks a single document by exact key match (O(log n))
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">Key value to seek</param>
|
/// <param name="key">Key value to seek</param>
|
||||||
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
||||||
@@ -219,68 +223,67 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
public DocumentLocation? Seek(object key, ITransaction? transaction = null)
|
public DocumentLocation? Seek(object key, ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
if (key == null)
|
if (key == null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
if (_vectorIndex != null && key is float[] query)
|
|
||||||
{
|
|
||||||
return _vectorIndex.Search(query, 1, transaction: transaction).FirstOrDefault().Location;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_btreeIndex != null)
|
if (_vectorIndex != null && key is float[] query)
|
||||||
|
return _vectorIndex.Search(query, 1, transaction: transaction).FirstOrDefault().Location;
|
||||||
|
|
||||||
|
if (BTreeIndex != null)
|
||||||
{
|
{
|
||||||
var userKey = ConvertToIndexKey(key);
|
var userKey = ConvertToIndexKey(key);
|
||||||
var minComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: true);
|
var minComposite = CreateCompositeKeyBoundary(userKey, true);
|
||||||
var maxComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: false);
|
var maxComposite = CreateCompositeKeyBoundary(userKey, false);
|
||||||
var firstEntry = _btreeIndex.Range(minComposite, maxComposite, IndexDirection.Forward, transaction?.TransactionId).FirstOrDefault();
|
var firstEntry = BTreeIndex
|
||||||
return firstEntry.Location.PageId == 0 ? null : (DocumentLocation?)firstEntry.Location;
|
.Range(minComposite, maxComposite, IndexDirection.Forward, transaction?.TransactionId).FirstOrDefault();
|
||||||
}
|
return firstEntry.Location.PageId == 0 ? null : firstEntry.Location;
|
||||||
|
}
|
||||||
return null;
|
|
||||||
}
|
return null;
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Performs a vector nearest-neighbor search.
|
/// <summary>
|
||||||
/// </summary>
|
/// Performs a vector nearest-neighbor search.
|
||||||
/// <param name="query">The query vector.</param>
|
/// </summary>
|
||||||
/// <param name="k">The number of results to return.</param>
|
/// <param name="query">The query vector.</param>
|
||||||
/// <param name="efSearch">The search breadth parameter.</param>
|
/// <param name="k">The number of results to return.</param>
|
||||||
/// <param name="transaction">Optional transaction.</param>
|
/// <param name="efSearch">The search breadth parameter.</param>
|
||||||
/// <returns>The matching vector search results.</returns>
|
/// <param name="transaction">Optional transaction.</param>
|
||||||
public IEnumerable<VectorSearchResult> VectorSearch(float[] query, int k, int efSearch = 100, ITransaction? transaction = null)
|
/// <returns>The matching vector search results.</returns>
|
||||||
{
|
public IEnumerable<VectorSearchResult> VectorSearch(float[] query, int k, int efSearch = 100,
|
||||||
if (_vectorIndex == null)
|
ITransaction? transaction = null)
|
||||||
throw new InvalidOperationException("This index is not a vector index.");
|
{
|
||||||
|
if (_vectorIndex == null)
|
||||||
|
throw new InvalidOperationException("This index is not a vector index.");
|
||||||
|
|
||||||
return _vectorIndex.Search(query, k, efSearch, transaction);
|
return _vectorIndex.Search(query, k, efSearch, transaction);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs geospatial distance search
|
/// Performs geospatial distance search
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="center">The center point.</param>
|
/// <param name="center">The center point.</param>
|
||||||
/// <param name="radiusKm">The search radius in kilometers.</param>
|
/// <param name="radiusKm">The search radius in kilometers.</param>
|
||||||
/// <param name="transaction">Optional transaction.</param>
|
/// <param name="transaction">Optional transaction.</param>
|
||||||
public IEnumerable<DocumentLocation> Near((double Latitude, double Longitude) center, double radiusKm, ITransaction? transaction = null)
|
public IEnumerable<DocumentLocation> Near((double Latitude, double Longitude) center, double radiusKm,
|
||||||
{
|
ITransaction? transaction = null)
|
||||||
if (_spatialIndex == null)
|
{
|
||||||
|
if (_spatialIndex == null)
|
||||||
throw new InvalidOperationException("This index is not a spatial index.");
|
throw new InvalidOperationException("This index is not a spatial index.");
|
||||||
|
|
||||||
var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
||||||
foreach (var loc in _spatialIndex.Search(queryBox, transaction))
|
foreach (var loc in _spatialIndex.Search(queryBox, transaction)) yield return loc;
|
||||||
{
|
|
||||||
yield return loc;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs geospatial bounding box search
|
/// Performs geospatial bounding box search
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="min">The minimum latitude/longitude corner.</param>
|
/// <param name="min">The minimum latitude/longitude corner.</param>
|
||||||
/// <param name="max">The maximum latitude/longitude corner.</param>
|
/// <param name="max">The maximum latitude/longitude corner.</param>
|
||||||
/// <param name="transaction">Optional transaction.</param>
|
/// <param name="transaction">Optional transaction.</param>
|
||||||
public IEnumerable<DocumentLocation> Within((double Latitude, double Longitude) min, (double Latitude, double Longitude) max, ITransaction? transaction = null)
|
public IEnumerable<DocumentLocation> Within((double Latitude, double Longitude) min,
|
||||||
{
|
(double Latitude, double Longitude) max, ITransaction? transaction = null)
|
||||||
if (_spatialIndex == null)
|
{
|
||||||
|
if (_spatialIndex == null)
|
||||||
throw new InvalidOperationException("This index is not a spatial index.");
|
throw new InvalidOperationException("This index is not a spatial index.");
|
||||||
|
|
||||||
var area = new GeoBox(min.Latitude, min.Longitude, max.Latitude, max.Longitude);
|
var area = new GeoBox(min.Latitude, min.Longitude, max.Latitude, max.Longitude);
|
||||||
@@ -288,21 +291,22 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Scans a range of keys (O(log n + k) where k is result count)
|
/// Scans a range of keys (O(log n + k) where k is result count)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="minKey">Minimum key (inclusive), null for unbounded</param>
|
/// <param name="minKey">Minimum key (inclusive), null for unbounded</param>
|
||||||
/// <param name="maxKey">Maximum key (inclusive), null for unbounded</param>
|
/// <param name="maxKey">Maximum key (inclusive), null for unbounded</param>
|
||||||
/// <param name="direction">Scan direction.</param>
|
/// <param name="direction">Scan direction.</param>
|
||||||
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
/// <param name="transaction">Optional transaction to read uncommitted changes</param>
|
||||||
/// <returns>Enumerable of document locations in key order</returns>
|
/// <returns>Enumerable of document locations in key order</returns>
|
||||||
public IEnumerable<DocumentLocation> Range(object? minKey, object? maxKey, IndexDirection direction = IndexDirection.Forward, ITransaction? transaction = null)
|
public IEnumerable<DocumentLocation> Range(object? minKey, object? maxKey,
|
||||||
|
IndexDirection direction = IndexDirection.Forward, ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
if (_btreeIndex == null) yield break;
|
if (BTreeIndex == null) yield break;
|
||||||
|
|
||||||
// Handle unbounded ranges
|
// Handle unbounded ranges
|
||||||
IndexKey actualMinKey;
|
IndexKey actualMinKey;
|
||||||
IndexKey actualMaxKey;
|
IndexKey actualMaxKey;
|
||||||
|
|
||||||
if (minKey == null && maxKey == null)
|
if (minKey == null && maxKey == null)
|
||||||
{
|
{
|
||||||
// Full scan - use extreme values
|
// Full scan - use extreme values
|
||||||
@@ -313,108 +317,53 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
{
|
{
|
||||||
actualMinKey = new IndexKey(new byte[0]);
|
actualMinKey = new IndexKey(new byte[0]);
|
||||||
var userMaxKey = ConvertToIndexKey(maxKey!);
|
var userMaxKey = ConvertToIndexKey(maxKey!);
|
||||||
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false); // Max boundary
|
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, false); // Max boundary
|
||||||
}
|
}
|
||||||
else if (maxKey == null)
|
else if (maxKey == null)
|
||||||
{
|
{
|
||||||
var userMinKey = ConvertToIndexKey(minKey);
|
var userMinKey = ConvertToIndexKey(minKey);
|
||||||
actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true); // Min boundary
|
actualMinKey = CreateCompositeKeyBoundary(userMinKey, true); // Min boundary
|
||||||
actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray());
|
actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray());
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Both bounds specified
|
// Both bounds specified
|
||||||
var userMinKey = ConvertToIndexKey(minKey);
|
var userMinKey = ConvertToIndexKey(minKey);
|
||||||
var userMaxKey = ConvertToIndexKey(maxKey);
|
var userMaxKey = ConvertToIndexKey(maxKey);
|
||||||
|
|
||||||
// Create composite boundaries:
|
// Create composite boundaries:
|
||||||
// Min: (userMinKey, ObjectId.Empty) - captures all docs with key >= userMinKey
|
// Min: (userMinKey, ObjectId.Empty) - captures all docs with key >= userMinKey
|
||||||
// Max: (userMaxKey, ObjectId.MaxValue) - captures all docs with key <= userMaxKey
|
// Max: (userMaxKey, ObjectId.MaxValue) - captures all docs with key <= userMaxKey
|
||||||
actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true);
|
actualMinKey = CreateCompositeKeyBoundary(userMinKey, true);
|
||||||
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false);
|
actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, false);
|
||||||
}
|
|
||||||
|
|
||||||
// Use BTreeIndex.Range with WAL-aware reads and direction
|
|
||||||
// Extract DocumentLocation from each entry
|
|
||||||
foreach (var entry in _btreeIndex.Range(actualMinKey, actualMaxKey, direction, transaction?.TransactionId))
|
|
||||||
{
|
|
||||||
yield return entry.Location;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Use BTreeIndex.Range with WAL-aware reads and direction
|
||||||
|
// Extract DocumentLocation from each entry
|
||||||
|
foreach (var entry in BTreeIndex.Range(actualMinKey, actualMaxKey, direction, transaction?.TransactionId))
|
||||||
|
yield return entry.Location;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets statistics about this index
|
/// Gets statistics about this index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CollectionIndexInfo GetInfo()
|
public CollectionIndexInfo GetInfo()
|
||||||
{
|
{
|
||||||
return new CollectionIndexInfo
|
return new CollectionIndexInfo
|
||||||
{
|
{
|
||||||
Name = _definition.Name,
|
Name = Definition.Name,
|
||||||
PropertyPaths = _definition.PropertyPaths,
|
PropertyPaths = Definition.PropertyPaths,
|
||||||
IsUnique = _definition.IsUnique,
|
IsUnique = Definition.IsUnique,
|
||||||
Type = _definition.Type,
|
Type = Definition.Type,
|
||||||
IsPrimary = _definition.IsPrimary,
|
IsPrimary = Definition.IsPrimary,
|
||||||
EstimatedDocumentCount = 0, // TODO: Track or calculate document count
|
EstimatedDocumentCount = 0, // TODO: Track or calculate document count
|
||||||
EstimatedSizeBytes = 0 // TODO: Calculate index size
|
EstimatedSizeBytes = 0 // TODO: Calculate index size
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
#region Composite Key Support (SQLite-style for Duplicate Keys)
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a composite key by concatenating user key with document ID.
|
/// Converts a CLR value to an IndexKey for BTree storage.
|
||||||
/// This allows duplicate user keys while maintaining BTree uniqueness.
|
/// Supports all common .NET types.
|
||||||
/// Format: [UserKeyBytes] + [DocumentIdKey]
|
|
||||||
/// </summary>
|
|
||||||
private IndexKey CreateCompositeKey(IndexKey userKey, IndexKey documentIdKey)
|
|
||||||
{
|
|
||||||
// Allocate buffer: user key + document ID key length
|
|
||||||
var compositeBytes = new byte[userKey.Data.Length + documentIdKey.Data.Length];
|
|
||||||
|
|
||||||
// Copy user key
|
|
||||||
userKey.Data.CopyTo(compositeBytes.AsSpan(0, userKey.Data.Length));
|
|
||||||
|
|
||||||
// Append document ID key
|
|
||||||
documentIdKey.Data.CopyTo(compositeBytes.AsSpan(userKey.Data.Length));
|
|
||||||
|
|
||||||
return new IndexKey(compositeBytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Creates a composite key for range query boundary.
|
|
||||||
/// Uses MIN or MAX ID representation to capture all documents with the user key.
|
|
||||||
/// </summary>
|
|
||||||
private IndexKey CreateCompositeKeyBoundary(IndexKey userKey, bool useMinObjectId)
|
|
||||||
{
|
|
||||||
// For range boundaries, we use an empty key for Min and a very large key for Max
|
|
||||||
// to wrap around all possible IDs for this user key.
|
|
||||||
IndexKey idBoundary = useMinObjectId
|
|
||||||
? new IndexKey(Array.Empty<byte>())
|
|
||||||
: new IndexKey(Enumerable.Repeat((byte)0xFF, 16).ToArray()); // Using 16 as a safe max for GUID/ObjectId
|
|
||||||
|
|
||||||
return CreateCompositeKey(userKey, idBoundary);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Extracts the original user key from a composite key by removing the ObjectId suffix.
|
|
||||||
/// Used when we need to return the original indexed value.
|
|
||||||
/// </summary>
|
|
||||||
private IndexKey ExtractUserKey(IndexKey compositeKey)
|
|
||||||
{
|
|
||||||
// Composite key = UserKey + ObjectId(12 bytes)
|
|
||||||
var userKeyLength = compositeKey.Data.Length - 12;
|
|
||||||
if (userKeyLength <= 0)
|
|
||||||
return compositeKey; // Fallback for malformed keys
|
|
||||||
|
|
||||||
var userKeyBytes = compositeKey.Data.Slice(0, userKeyLength);
|
|
||||||
return new IndexKey(userKeyBytes);
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Converts a CLR value to an IndexKey for BTree storage.
|
|
||||||
/// Supports all common .NET types.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private IndexKey ConvertToIndexKey(object value)
|
private IndexKey ConvertToIndexKey(object value)
|
||||||
{
|
{
|
||||||
@@ -426,26 +375,64 @@ public sealed class CollectionSecondaryIndex<TId, T> : IDisposable where T : cla
|
|||||||
long longVal => new IndexKey(longVal),
|
long longVal => new IndexKey(longVal),
|
||||||
DateTime dateTime => new IndexKey(dateTime.Ticks),
|
DateTime dateTime => new IndexKey(dateTime.Ticks),
|
||||||
bool boolVal => new IndexKey(boolVal ? 1 : 0),
|
bool boolVal => new IndexKey(boolVal ? 1 : 0),
|
||||||
byte[] byteArray => new IndexKey(byteArray),
|
byte[] byteArray => new IndexKey(byteArray),
|
||||||
|
|
||||||
// For compound keys or complex types, use ToString and serialize
|
// For compound keys or complex types, use ToString and serialize
|
||||||
// TODO: Better compound key serialization
|
// TODO: Better compound key serialization
|
||||||
_ => new IndexKey(value.ToString() ?? string.Empty)
|
_ => new IndexKey(value.ToString() ?? string.Empty)
|
||||||
};
|
};
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Releases resources used by this index wrapper.
|
|
||||||
/// </summary>
|
|
||||||
public void Dispose()
|
|
||||||
{
|
|
||||||
if (_disposed)
|
|
||||||
return;
|
|
||||||
|
|
||||||
// BTreeIndex doesn't currently implement IDisposable
|
|
||||||
// Future: may need to flush buffers, close resources
|
|
||||||
|
|
||||||
_disposed = true;
|
|
||||||
GC.SuppressFinalize(this);
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
#region Composite Key Support (SQLite-style for Duplicate Keys)
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a composite key by concatenating user key with document ID.
|
||||||
|
/// This allows duplicate user keys while maintaining BTree uniqueness.
|
||||||
|
/// Format: [UserKeyBytes] + [DocumentIdKey]
|
||||||
|
/// </summary>
|
||||||
|
private IndexKey CreateCompositeKey(IndexKey userKey, IndexKey documentIdKey)
|
||||||
|
{
|
||||||
|
// Allocate buffer: user key + document ID key length
|
||||||
|
var compositeBytes = new byte[userKey.Data.Length + documentIdKey.Data.Length];
|
||||||
|
|
||||||
|
// Copy user key
|
||||||
|
userKey.Data.CopyTo(compositeBytes.AsSpan(0, userKey.Data.Length));
|
||||||
|
|
||||||
|
// Append document ID key
|
||||||
|
documentIdKey.Data.CopyTo(compositeBytes.AsSpan(userKey.Data.Length));
|
||||||
|
|
||||||
|
return new IndexKey(compositeBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Creates a composite key for range query boundary.
|
||||||
|
/// Uses MIN or MAX ID representation to capture all documents with the user key.
|
||||||
|
/// </summary>
|
||||||
|
private IndexKey CreateCompositeKeyBoundary(IndexKey userKey, bool useMinObjectId)
|
||||||
|
{
|
||||||
|
// For range boundaries, we use an empty key for Min and a very large key for Max
|
||||||
|
// to wrap around all possible IDs for this user key.
|
||||||
|
var idBoundary = useMinObjectId
|
||||||
|
? new IndexKey(Array.Empty<byte>())
|
||||||
|
: new IndexKey(Enumerable.Repeat((byte)0xFF, 16).ToArray()); // Using 16 as a safe max for GUID/ObjectId
|
||||||
|
|
||||||
|
return CreateCompositeKey(userKey, idBoundary);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extracts the original user key from a composite key by removing the ObjectId suffix.
|
||||||
|
/// Used when we need to return the original indexed value.
|
||||||
|
/// </summary>
|
||||||
|
private IndexKey ExtractUserKey(IndexKey compositeKey)
|
||||||
|
{
|
||||||
|
// Composite key = UserKey + ObjectId(12 bytes)
|
||||||
|
int userKeyLength = compositeKey.Data.Length - 12;
|
||||||
|
if (userKeyLength <= 0)
|
||||||
|
return compositeKey; // Fallback for malformed keys
|
||||||
|
|
||||||
|
var userKeyBytes = compositeKey.Data.Slice(0, userKeyLength);
|
||||||
|
return new IndexKey(userKeyBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
@@ -3,28 +3,30 @@ namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
|||||||
public static class GeoSpatialExtensions
|
public static class GeoSpatialExtensions
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a geospatial proximity search (Near) on a coordinate tuple property.
|
/// Performs a geospatial proximity search (Near) on a coordinate tuple property.
|
||||||
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
||||||
/// <param name="center">The center point (Latitude, Longitude) for the proximity search.</param>
|
/// <param name="center">The center point (Latitude, Longitude) for the proximity search.</param>
|
||||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||||
/// <returns>True if the point is within the specified radius.</returns>
|
/// <returns>True if the point is within the specified radius.</returns>
|
||||||
public static bool Near(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) center, double radiusKm)
|
public static bool Near(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) center,
|
||||||
|
double radiusKm)
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a geospatial bounding box search (Within) on a coordinate tuple property.
|
/// Performs a geospatial bounding box search (Within) on a coordinate tuple property.
|
||||||
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
/// This method is a marker for the LINQ query provider and is optimized using R-Tree indexes if available.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
/// <param name="point">The coordinate tuple (Latitude, Longitude) property of the entity.</param>
|
||||||
/// <param name="min">The minimum (Latitude, Longitude) of the bounding box.</param>
|
/// <param name="min">The minimum (Latitude, Longitude) of the bounding box.</param>
|
||||||
/// <param name="max">The maximum (Latitude, Longitude) of the bounding box.</param>
|
/// <param name="max">The maximum (Latitude, Longitude) of the bounding box.</param>
|
||||||
/// <returns>True if the point is within the specified bounding box.</returns>
|
/// <returns>True if the point is within the specified bounding box.</returns>
|
||||||
public static bool Within(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) min, (double Latitude, double Longitude) max)
|
public static bool Within(this (double Latitude, double Longitude) point, (double Latitude, double Longitude) min,
|
||||||
|
(double Latitude, double Longitude) max)
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,13 +1,10 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Hash-based index for exact-match lookups.
|
/// Hash-based index for exact-match lookups.
|
||||||
/// Uses simple bucket-based hashing with collision handling.
|
/// Uses simple bucket-based hashing with collision handling.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class HashIndex
|
public sealed class HashIndex
|
||||||
{
|
{
|
||||||
@@ -15,7 +12,7 @@ public sealed class HashIndex
|
|||||||
private readonly IndexOptions _options;
|
private readonly IndexOptions _options;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="HashIndex"/> class.
|
/// Initializes a new instance of the <see cref="HashIndex" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">The index options.</param>
|
/// <param name="options">The index options.</param>
|
||||||
public HashIndex(IndexOptions options)
|
public HashIndex(IndexOptions options)
|
||||||
@@ -25,16 +22,16 @@ public sealed class HashIndex
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a key-location pair into the hash index
|
/// Inserts a key-location pair into the hash index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <param name="location">The document location.</param>
|
/// <param name="location">The document location.</param>
|
||||||
public void Insert(IndexKey key, DocumentLocation location)
|
public void Insert(IndexKey key, DocumentLocation location)
|
||||||
{
|
{
|
||||||
if (_options.Unique && TryFind(key, out _))
|
if (_options.Unique && TryFind(key, out _))
|
||||||
throw new InvalidOperationException($"Duplicate key violation for unique index");
|
throw new InvalidOperationException("Duplicate key violation for unique index");
|
||||||
|
|
||||||
var hashCode = key.GetHashCode();
|
int hashCode = key.GetHashCode();
|
||||||
|
|
||||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||||
{
|
{
|
||||||
@@ -46,46 +43,43 @@ public sealed class HashIndex
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Finds a document location by exact key match
|
/// Finds a document location by exact key match
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <param name="location">When this method returns, contains the matched document location if found.</param>
|
/// <param name="location">When this method returns, contains the matched document location if found.</param>
|
||||||
/// <returns><see langword="true"/> if a matching entry is found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if a matching entry is found; otherwise, <see langword="false" />.</returns>
|
||||||
public bool TryFind(IndexKey key, out DocumentLocation location)
|
public bool TryFind(IndexKey key, out DocumentLocation location)
|
||||||
{
|
{
|
||||||
location = default;
|
location = default;
|
||||||
var hashCode = key.GetHashCode();
|
int hashCode = key.GetHashCode();
|
||||||
|
|
||||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
foreach (var entry in bucket)
|
foreach (var entry in bucket)
|
||||||
{
|
|
||||||
if (entry.Key == key)
|
if (entry.Key == key)
|
||||||
{
|
{
|
||||||
location = entry.Location;
|
location = entry.Location;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Removes an entry from the index
|
/// Removes an entry from the index
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <param name="location">The document location.</param>
|
/// <param name="location">The document location.</param>
|
||||||
/// <returns><see langword="true"/> if an entry is removed; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if an entry is removed; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Remove(IndexKey key, DocumentLocation location)
|
public bool Remove(IndexKey key, DocumentLocation location)
|
||||||
{
|
{
|
||||||
var hashCode = key.GetHashCode();
|
int hashCode = key.GetHashCode();
|
||||||
|
|
||||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
for (int i = 0; i < bucket.Count; i++)
|
for (var i = 0; i < bucket.Count; i++)
|
||||||
{
|
|
||||||
if (bucket[i].Key == key &&
|
if (bucket[i].Key == key &&
|
||||||
bucket[i].Location.PageId == location.PageId &&
|
bucket[i].Location.PageId == location.PageId &&
|
||||||
bucket[i].Location.SlotIndex == location.SlotIndex)
|
bucket[i].Location.SlotIndex == location.SlotIndex)
|
||||||
@@ -97,27 +91,24 @@ public sealed class HashIndex
|
|||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets all entries matching the key
|
/// Gets all entries matching the key
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The index key.</param>
|
/// <param name="key">The index key.</param>
|
||||||
/// <returns>All matching index entries.</returns>
|
/// <returns>All matching index entries.</returns>
|
||||||
public IEnumerable<IndexEntry> FindAll(IndexKey key)
|
public IEnumerable<IndexEntry> FindAll(IndexKey key)
|
||||||
{
|
{
|
||||||
var hashCode = key.GetHashCode();
|
int hashCode = key.GetHashCode();
|
||||||
|
|
||||||
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
if (!_buckets.TryGetValue(hashCode, out var bucket))
|
||||||
yield break;
|
yield break;
|
||||||
|
|
||||||
foreach (var entry in bucket)
|
foreach (var entry in bucket)
|
||||||
{
|
|
||||||
if (entry.Key == key)
|
if (entry.Key == key)
|
||||||
yield return entry;
|
yield return entry;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,50 +1,47 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
using System;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a cursor for traversing a B+Tree index.
|
/// Represents a cursor for traversing a B+Tree index.
|
||||||
/// Provides low-level primitives for building complex queries.
|
/// Provides low-level primitives for building complex queries.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public interface IBTreeCursor : IDisposable
|
public interface IBTreeCursor : IDisposable
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current entry at the cursor position.
|
/// Gets the current entry at the cursor position.
|
||||||
/// Throws InvalidOperationException if cursor is invalid or uninitialized.
|
/// Throws InvalidOperationException if cursor is invalid or uninitialized.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
IndexEntry Current { get; }
|
IndexEntry Current { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the first entry in the index.
|
/// Moves the cursor to the first entry in the index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
||||||
bool MoveToFirst();
|
bool MoveToFirst();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the last entry in the index.
|
/// Moves the cursor to the last entry in the index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
/// <returns>True if the index is not empty; otherwise, false.</returns>
|
||||||
bool MoveToLast();
|
bool MoveToLast();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Seeks to the specified key.
|
/// Seeks to the specified key.
|
||||||
/// If exact match found, positions there and returns true.
|
/// If exact match found, positions there and returns true.
|
||||||
/// If not found, positions at the next greater key and returns false.
|
/// If not found, positions at the next greater key and returns false.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">Key to seek</param>
|
/// <param name="key">Key to seek</param>
|
||||||
/// <returns>True if exact match found; false if positioned at next greater key.</returns>
|
/// <returns>True if exact match found; false if positioned at next greater key.</returns>
|
||||||
bool Seek(IndexKey key);
|
bool Seek(IndexKey key);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Advances the cursor to the next entry.
|
/// Advances the cursor to the next entry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>True if successfully moved; false if end of index reached.</returns>
|
/// <returns>True if successfully moved; false if end of index reached.</returns>
|
||||||
bool MoveNext();
|
bool MoveNext();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Moves the cursor to the previous entry.
|
/// Moves the cursor to the previous entry.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>True if successfully moved; false if start of index reached.</returns>
|
/// <returns>True if successfully moved; false if start of index reached.</returns>
|
||||||
bool MovePrev();
|
bool MovePrev();
|
||||||
}
|
}
|
||||||
@@ -4,4 +4,4 @@ public enum IndexDirection
|
|||||||
{
|
{
|
||||||
Forward,
|
Forward,
|
||||||
Backward
|
Backward
|
||||||
}
|
}
|
||||||
@@ -1,31 +1,30 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using System.Text;
|
||||||
using System;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using System.Linq;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
|
/// <summary>
|
||||||
/// <summary>
|
/// Represents a key in an index.
|
||||||
/// Represents a key in an index.
|
/// Implemented as struct for efficient index operations.
|
||||||
/// Implemented as struct for efficient index operations.
|
/// Note: Contains byte array so cannot be readonly struct.
|
||||||
/// Note: Contains byte array so cannot be readonly struct.
|
/// </summary>
|
||||||
/// </summary>
|
public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
||||||
public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
{
|
||||||
{
|
private readonly byte[] _data;
|
||||||
private readonly byte[] _data;
|
private readonly int _hashCode;
|
||||||
private readonly int _hashCode;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the minimum possible index key.
|
|
||||||
/// </summary>
|
|
||||||
public static IndexKey MinKey => new IndexKey(Array.Empty<byte>());
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the maximum possible index key.
|
/// Gets the minimum possible index key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static IndexKey MaxKey => new IndexKey(Enumerable.Repeat((byte)0xFF, 32).ToArray());
|
public static IndexKey MinKey => new(Array.Empty<byte>());
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from raw key bytes.
|
/// Gets the maximum possible index key.
|
||||||
|
/// </summary>
|
||||||
|
public static IndexKey MaxKey => new(Enumerable.Repeat((byte)0xFF, 32).ToArray());
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from raw key bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="data">The key bytes.</param>
|
/// <param name="data">The key bytes.</param>
|
||||||
public IndexKey(ReadOnlySpan<byte> data)
|
public IndexKey(ReadOnlySpan<byte> data)
|
||||||
@@ -35,7 +34,7 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from an object identifier.
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from an object identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="objectId">The object identifier value.</param>
|
/// <param name="objectId">The object identifier value.</param>
|
||||||
public IndexKey(ObjectId objectId)
|
public IndexKey(ObjectId objectId)
|
||||||
@@ -46,7 +45,7 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a 32-bit integer.
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from a 32-bit integer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="value">The integer value.</param>
|
/// <param name="value">The integer value.</param>
|
||||||
public IndexKey(int value)
|
public IndexKey(int value)
|
||||||
@@ -56,7 +55,7 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a 64-bit integer.
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from a 64-bit integer.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="value">The integer value.</param>
|
/// <param name="value">The integer value.</param>
|
||||||
public IndexKey(long value)
|
public IndexKey(long value)
|
||||||
@@ -66,17 +65,17 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a string.
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from a string.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="value">The string value.</param>
|
/// <param name="value">The string value.</param>
|
||||||
public IndexKey(string value)
|
public IndexKey(string value)
|
||||||
{
|
{
|
||||||
_data = System.Text.Encoding.UTF8.GetBytes(value);
|
_data = Encoding.UTF8.GetBytes(value);
|
||||||
_hashCode = ComputeHashCode(_data);
|
_hashCode = ComputeHashCode(_data);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="IndexKey"/> struct from a GUID.
|
/// Initializes a new instance of the <see cref="IndexKey" /> struct from a GUID.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="value">The GUID value.</param>
|
/// <param name="value">The GUID value.</param>
|
||||||
public IndexKey(Guid value)
|
public IndexKey(Guid value)
|
||||||
@@ -86,72 +85,102 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the raw byte data for this key.
|
/// Gets the raw byte data for this key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly ReadOnlySpan<byte> Data => _data;
|
public readonly ReadOnlySpan<byte> Data => _data;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compares this key to another key.
|
/// Compares this key to another key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The key to compare with.</param>
|
/// <param name="other">The key to compare with.</param>
|
||||||
/// <returns>
|
/// <returns>
|
||||||
/// A value less than zero if this key is less than <paramref name="other"/>, zero if equal, or greater than zero if greater.
|
/// A value less than zero if this key is less than <paramref name="other" />, zero if equal, or greater than zero if
|
||||||
|
/// greater.
|
||||||
/// </returns>
|
/// </returns>
|
||||||
public readonly int CompareTo(IndexKey other)
|
public readonly int CompareTo(IndexKey other)
|
||||||
{
|
{
|
||||||
if (_data == null) return other._data == null ? 0 : -1;
|
if (_data == null) return other._data == null ? 0 : -1;
|
||||||
if (other._data == null) return 1;
|
if (other._data == null) return 1;
|
||||||
|
|
||||||
var minLength = Math.Min(_data.Length, other._data.Length);
|
|
||||||
|
|
||||||
for (int i = 0; i < minLength; i++)
|
int minLength = Math.Min(_data.Length, other._data.Length);
|
||||||
{
|
|
||||||
var cmp = _data[i].CompareTo(other._data[i]);
|
for (var i = 0; i < minLength; i++)
|
||||||
if (cmp != 0)
|
{
|
||||||
return cmp;
|
int cmp = _data[i].CompareTo(other._data[i]);
|
||||||
}
|
if (cmp != 0)
|
||||||
|
return cmp;
|
||||||
|
}
|
||||||
|
|
||||||
return _data.Length.CompareTo(other._data.Length);
|
return _data.Length.CompareTo(other._data.Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this key equals another key.
|
/// Determines whether this key equals another key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The key to compare with.</param>
|
/// <param name="other">The key to compare with.</param>
|
||||||
/// <returns><see langword="true"/> if the keys are equal; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the keys are equal; otherwise, <see langword="false" />.</returns>
|
||||||
public readonly bool Equals(IndexKey other)
|
public readonly bool Equals(IndexKey other)
|
||||||
{
|
{
|
||||||
if (_hashCode != other._hashCode)
|
if (_hashCode != other._hashCode)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (_data == null) return other._data == null;
|
if (_data == null) return other._data == null;
|
||||||
if (other._data == null) return false;
|
if (other._data == null) return false;
|
||||||
|
|
||||||
return _data.AsSpan().SequenceEqual(other._data);
|
return _data.AsSpan().SequenceEqual(other._data);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override readonly bool Equals(object? obj) => obj is IndexKey other && Equals(other);
|
public readonly override bool Equals(object? obj)
|
||||||
|
{
|
||||||
|
return obj is IndexKey other && Equals(other);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override readonly int GetHashCode() => _hashCode;
|
public readonly override int GetHashCode()
|
||||||
|
{
|
||||||
public static bool operator ==(IndexKey left, IndexKey right) => left.Equals(right);
|
return _hashCode;
|
||||||
public static bool operator !=(IndexKey left, IndexKey right) => !left.Equals(right);
|
}
|
||||||
public static bool operator <(IndexKey left, IndexKey right) => left.CompareTo(right) < 0;
|
|
||||||
public static bool operator >(IndexKey left, IndexKey right) => left.CompareTo(right) > 0;
|
public static bool operator ==(IndexKey left, IndexKey right)
|
||||||
public static bool operator <=(IndexKey left, IndexKey right) => left.CompareTo(right) <= 0;
|
{
|
||||||
public static bool operator >=(IndexKey left, IndexKey right) => left.CompareTo(right) >= 0;
|
return left.Equals(right);
|
||||||
|
}
|
||||||
private static int ComputeHashCode(ReadOnlySpan<byte> data)
|
|
||||||
{
|
public static bool operator !=(IndexKey left, IndexKey right)
|
||||||
var hash = new HashCode();
|
{
|
||||||
hash.AddBytes(data);
|
return !left.Equals(right);
|
||||||
return hash.ToHashCode();
|
}
|
||||||
}
|
|
||||||
|
public static bool operator <(IndexKey left, IndexKey right)
|
||||||
|
{
|
||||||
|
return left.CompareTo(right) < 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator >(IndexKey left, IndexKey right)
|
||||||
|
{
|
||||||
|
return left.CompareTo(right) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator <=(IndexKey left, IndexKey right)
|
||||||
|
{
|
||||||
|
return left.CompareTo(right) <= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static bool operator >=(IndexKey left, IndexKey right)
|
||||||
|
{
|
||||||
|
return left.CompareTo(right) >= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ComputeHashCode(ReadOnlySpan<byte> data)
|
||||||
|
{
|
||||||
|
var hash = new HashCode();
|
||||||
|
hash.AddBytes(data);
|
||||||
|
return hash.ToHashCode();
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates an <see cref="IndexKey"/> from a supported CLR value.
|
/// Creates an <see cref="IndexKey" /> from a supported CLR value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="T">The CLR type of the value.</typeparam>
|
/// <typeparam name="T">The CLR type of the value.</typeparam>
|
||||||
/// <param name="value">The value to convert.</param>
|
/// <param name="value">The value to convert.</param>
|
||||||
@@ -159,33 +188,35 @@ public struct IndexKey : IEquatable<IndexKey>, IComparable<IndexKey>
|
|||||||
public static IndexKey Create<T>(T value)
|
public static IndexKey Create<T>(T value)
|
||||||
{
|
{
|
||||||
if (value == null) return default;
|
if (value == null) return default;
|
||||||
|
|
||||||
if (typeof(T) == typeof(ObjectId)) return new IndexKey((ObjectId)(object)value);
|
if (typeof(T) == typeof(ObjectId)) return new IndexKey((ObjectId)(object)value);
|
||||||
if (typeof(T) == typeof(int)) return new IndexKey((int)(object)value);
|
if (typeof(T) == typeof(int)) return new IndexKey((int)(object)value);
|
||||||
if (typeof(T) == typeof(long)) return new IndexKey((long)(object)value);
|
if (typeof(T) == typeof(long)) return new IndexKey((long)(object)value);
|
||||||
if (typeof(T) == typeof(string)) return new IndexKey((string)(object)value);
|
if (typeof(T) == typeof(string)) return new IndexKey((string)(object)value);
|
||||||
if (typeof(T) == typeof(Guid)) return new IndexKey((Guid)(object)value);
|
if (typeof(T) == typeof(Guid)) return new IndexKey((Guid)(object)value);
|
||||||
if (typeof(T) == typeof(byte[])) return new IndexKey((byte[])(object)value);
|
if (typeof(T) == typeof(byte[])) return new IndexKey((byte[])(object)value);
|
||||||
|
|
||||||
throw new NotSupportedException($"Type {typeof(T).Name} is not supported as an IndexKey. Provide a custom mapping.");
|
throw new NotSupportedException(
|
||||||
|
$"Type {typeof(T).Name} is not supported as an IndexKey. Provide a custom mapping.");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts this key to a CLR value of type <typeparamref name="T"/>.
|
/// Converts this key to a CLR value of type <typeparamref name="T" />.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="T">The CLR type to read from this key.</typeparam>
|
/// <typeparam name="T">The CLR type to read from this key.</typeparam>
|
||||||
/// <returns>The converted value.</returns>
|
/// <returns>The converted value.</returns>
|
||||||
public readonly T As<T>()
|
public readonly T As<T>()
|
||||||
{
|
{
|
||||||
if (_data == null) return default!;
|
if (_data == null) return default!;
|
||||||
|
|
||||||
if (typeof(T) == typeof(ObjectId)) return (T)(object)new ObjectId(_data);
|
if (typeof(T) == typeof(ObjectId)) return (T)(object)new ObjectId(_data);
|
||||||
if (typeof(T) == typeof(int)) return (T)(object)BitConverter.ToInt32(_data);
|
if (typeof(T) == typeof(int)) return (T)(object)BitConverter.ToInt32(_data);
|
||||||
if (typeof(T) == typeof(long)) return (T)(object)BitConverter.ToInt64(_data);
|
if (typeof(T) == typeof(long)) return (T)(object)BitConverter.ToInt64(_data);
|
||||||
if (typeof(T) == typeof(string)) return (T)(object)System.Text.Encoding.UTF8.GetString(_data);
|
if (typeof(T) == typeof(string)) return (T)(object)Encoding.UTF8.GetString(_data);
|
||||||
if (typeof(T) == typeof(Guid)) return (T)(object)new Guid(_data);
|
if (typeof(T) == typeof(Guid)) return (T)(object)new Guid(_data);
|
||||||
if (typeof(T) == typeof(byte[])) return (T)(object)_data;
|
if (typeof(T) == typeof(byte[])) return (T)(object)_data;
|
||||||
|
|
||||||
throw new NotSupportedException($"Type {typeof(T).Name} cannot be extracted from IndexKey. Provide a custom mapping.");
|
throw new NotSupportedException(
|
||||||
}
|
$"Type {typeof(T).Name} cannot be extracted from IndexKey. Provide a custom mapping.");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
@@ -1,121 +1,130 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Types of indices supported
|
/// Types of indices supported
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum IndexType : byte
|
public enum IndexType : byte
|
||||||
{
|
{
|
||||||
/// <summary>B+Tree index for range queries and ordering</summary>
|
/// <summary>B+Tree index for range queries and ordering</summary>
|
||||||
BTree = 1,
|
BTree = 1,
|
||||||
|
|
||||||
/// <summary>Hash index for exact match lookups</summary>
|
/// <summary>Hash index for exact match lookups</summary>
|
||||||
Hash = 2,
|
Hash = 2,
|
||||||
|
|
||||||
/// <summary>Unique index constraint</summary>
|
/// <summary>Unique index constraint</summary>
|
||||||
Unique = 3,
|
Unique = 3,
|
||||||
|
|
||||||
/// <summary>Vector index (HNSW) for similarity search</summary>
|
/// <summary>Vector index (HNSW) for similarity search</summary>
|
||||||
Vector = 4,
|
Vector = 4,
|
||||||
|
|
||||||
/// <summary>Geospatial index (R-Tree) for spatial queries</summary>
|
/// <summary>Geospatial index (R-Tree) for spatial queries</summary>
|
||||||
Spatial = 5
|
Spatial = 5
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Distance metrics for vector search
|
/// Distance metrics for vector search
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum VectorMetric : byte
|
public enum VectorMetric : byte
|
||||||
{
|
{
|
||||||
/// <summary>Cosine Similarity (Standard for embeddings)</summary>
|
/// <summary>Cosine Similarity (Standard for embeddings)</summary>
|
||||||
Cosine = 1,
|
Cosine = 1,
|
||||||
|
|
||||||
/// <summary>Euclidean Distance (L2)</summary>
|
/// <summary>Euclidean Distance (L2)</summary>
|
||||||
L2 = 2,
|
L2 = 2,
|
||||||
|
|
||||||
/// <summary>Dot Product</summary>
|
/// <summary>Dot Product</summary>
|
||||||
DotProduct = 3
|
DotProduct = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Index options and configuration.
|
/// Index options and configuration.
|
||||||
/// Implemented as readonly struct for efficiency.
|
/// Implemented as readonly struct for efficiency.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly struct IndexOptions
|
public readonly struct IndexOptions
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured index type.
|
/// Gets the configured index type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexType Type { get; init; }
|
public IndexType Type { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether the index enforces uniqueness.
|
/// Gets a value indicating whether the index enforces uniqueness.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool Unique { get; init; }
|
public bool Unique { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the indexed field names.
|
/// Gets the indexed field names.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] Fields { get; init; }
|
public string[] Fields { get; init; }
|
||||||
|
|
||||||
// Vector search options
|
// Vector search options
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the vector dimensionality for vector indexes.
|
/// Gets the vector dimensionality for vector indexes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Dimensions { get; init; }
|
public int Dimensions { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the distance metric used for vector similarity.
|
/// Gets the distance metric used for vector similarity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public VectorMetric Metric { get; init; }
|
public VectorMetric Metric { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the minimum number of graph connections per node.
|
/// Gets the minimum number of graph connections per node.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int M { get; init; } // Min number of connections per node
|
public int M { get; init; } // Min number of connections per node
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the size of the dynamic candidate list during index construction.
|
/// Gets the size of the dynamic candidate list during index construction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int EfConstruction { get; init; } // Size of dynamic candidate list for construction
|
public int EfConstruction { get; init; } // Size of dynamic candidate list for construction
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates non-unique B+Tree index options.
|
/// Creates non-unique B+Tree index options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fields">The indexed field names.</param>
|
/// <param name="fields">The indexed field names.</param>
|
||||||
/// <returns>The configured index options.</returns>
|
/// <returns>The configured index options.</returns>
|
||||||
public static IndexOptions CreateBTree(params string[] fields) => new()
|
public static IndexOptions CreateBTree(params string[] fields)
|
||||||
{
|
{
|
||||||
Type = IndexType.BTree,
|
return new IndexOptions
|
||||||
Unique = false,
|
{
|
||||||
Fields = fields
|
Type = IndexType.BTree,
|
||||||
};
|
Unique = false,
|
||||||
|
Fields = fields
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates unique B+Tree index options.
|
/// Creates unique B+Tree index options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fields">The indexed field names.</param>
|
/// <param name="fields">The indexed field names.</param>
|
||||||
/// <returns>The configured index options.</returns>
|
/// <returns>The configured index options.</returns>
|
||||||
public static IndexOptions CreateUnique(params string[] fields) => new()
|
public static IndexOptions CreateUnique(params string[] fields)
|
||||||
{
|
{
|
||||||
Type = IndexType.BTree,
|
return new IndexOptions
|
||||||
Unique = true,
|
{
|
||||||
Fields = fields
|
Type = IndexType.BTree,
|
||||||
};
|
Unique = true,
|
||||||
|
Fields = fields
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates hash index options.
|
/// Creates hash index options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fields">The indexed field names.</param>
|
/// <param name="fields">The indexed field names.</param>
|
||||||
/// <returns>The configured index options.</returns>
|
/// <returns>The configured index options.</returns>
|
||||||
public static IndexOptions CreateHash(params string[] fields) => new()
|
public static IndexOptions CreateHash(params string[] fields)
|
||||||
{
|
{
|
||||||
Type = IndexType.Hash,
|
return new IndexOptions
|
||||||
Unique = false,
|
{
|
||||||
Fields = fields
|
Type = IndexType.Hash,
|
||||||
};
|
Unique = false,
|
||||||
|
Fields = fields
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates vector index options.
|
/// Creates vector index options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="dimensions">The vector dimensionality.</param>
|
/// <param name="dimensions">The vector dimensionality.</param>
|
||||||
/// <param name="metric">The similarity metric.</param>
|
/// <param name="metric">The similarity metric.</param>
|
||||||
@@ -123,26 +132,33 @@ public readonly struct IndexOptions
|
|||||||
/// <param name="ef">The candidate list size used during index construction.</param>
|
/// <param name="ef">The candidate list size used during index construction.</param>
|
||||||
/// <param name="fields">The indexed field names.</param>
|
/// <param name="fields">The indexed field names.</param>
|
||||||
/// <returns>The configured index options.</returns>
|
/// <returns>The configured index options.</returns>
|
||||||
public static IndexOptions CreateVector(int dimensions, VectorMetric metric = VectorMetric.Cosine, int m = 16, int ef = 200, params string[] fields) => new()
|
public static IndexOptions CreateVector(int dimensions, VectorMetric metric = VectorMetric.Cosine, int m = 16,
|
||||||
|
int ef = 200, params string[] fields)
|
||||||
{
|
{
|
||||||
Type = IndexType.Vector,
|
return new IndexOptions
|
||||||
Unique = false,
|
{
|
||||||
Fields = fields,
|
Type = IndexType.Vector,
|
||||||
Dimensions = dimensions,
|
Unique = false,
|
||||||
Metric = metric,
|
Fields = fields,
|
||||||
M = m,
|
Dimensions = dimensions,
|
||||||
EfConstruction = ef
|
Metric = metric,
|
||||||
};
|
M = m,
|
||||||
|
EfConstruction = ef
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates spatial index options.
|
/// Creates spatial index options.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="fields">The indexed field names.</param>
|
/// <param name="fields">The indexed field names.</param>
|
||||||
/// <returns>The configured index options.</returns>
|
/// <returns>The configured index options.</returns>
|
||||||
public static IndexOptions CreateSpatial(params string[] fields) => new()
|
public static IndexOptions CreateSpatial(params string[] fields)
|
||||||
{
|
{
|
||||||
Type = IndexType.Spatial,
|
return new IndexOptions
|
||||||
Unique = false,
|
{
|
||||||
Fields = fields
|
Type = IndexType.Spatial,
|
||||||
};
|
Unique = false,
|
||||||
}
|
Fields = fields
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,90 +1,90 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Basic spatial point (Latitude/Longitude)
|
/// Basic spatial point (Latitude/Longitude)
|
||||||
/// Internal primitive for R-Tree logic.
|
/// Internal primitive for R-Tree logic.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal record struct GeoPoint(double Latitude, double Longitude)
|
internal record struct GeoPoint(double Latitude, double Longitude)
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets an empty point at coordinate origin.
|
/// Gets an empty point at coordinate origin.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static GeoPoint Empty => new(0, 0);
|
public static GeoPoint Empty => new(0, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Minimum Bounding Box (MBR) for spatial indexing
|
/// Minimum Bounding Box (MBR) for spatial indexing
|
||||||
/// Internal primitive for R-Tree logic.
|
/// Internal primitive for R-Tree logic.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal record struct GeoBox(double MinLat, double MinLon, double MaxLat, double MaxLon)
|
internal record struct GeoBox(double MinLat, double MinLon, double MaxLat, double MaxLon)
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets an empty bounding box sentinel value.
|
/// Gets an empty bounding box sentinel value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static GeoBox Empty => new(double.MaxValue, double.MaxValue, double.MinValue, double.MinValue);
|
public static GeoBox Empty => new(double.MaxValue, double.MaxValue, double.MinValue, double.MinValue);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this box contains the specified point.
|
/// Gets the area of this bounding box.
|
||||||
|
/// </summary>
|
||||||
|
public double Area => Math.Max(0, MaxLat - MinLat) * Math.Max(0, MaxLon - MinLon);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether this box contains the specified point.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="point">The point to test.</param>
|
/// <param name="point">The point to test.</param>
|
||||||
/// <returns><see langword="true"/> if the point is inside this box; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the point is inside this box; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Contains(GeoPoint point)
|
public bool Contains(GeoPoint point)
|
||||||
{
|
{
|
||||||
return point.Latitude >= MinLat && point.Latitude <= MaxLat &&
|
return point.Latitude >= MinLat && point.Latitude <= MaxLat &&
|
||||||
point.Longitude >= MinLon && point.Longitude <= MaxLon;
|
point.Longitude >= MinLon && point.Longitude <= MaxLon;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether this box intersects another box.
|
/// Determines whether this box intersects another box.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The other box to test.</param>
|
/// <param name="other">The other box to test.</param>
|
||||||
/// <returns><see langword="true"/> if the boxes intersect; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the boxes intersect; otherwise, <see langword="false" />.</returns>
|
||||||
public bool Intersects(GeoBox other)
|
public bool Intersects(GeoBox other)
|
||||||
{
|
{
|
||||||
return !(other.MinLat > MaxLat || other.MaxLat < MinLat ||
|
return !(other.MinLat > MaxLat || other.MaxLat < MinLat ||
|
||||||
other.MinLon > MaxLon || other.MaxLon < MinLon);
|
other.MinLon > MaxLon || other.MaxLon < MinLon);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a box that contains a single point.
|
/// Creates a box that contains a single point.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="point">The point to convert.</param>
|
/// <param name="point">The point to convert.</param>
|
||||||
/// <returns>A bounding box containing the specified point.</returns>
|
/// <returns>A bounding box containing the specified point.</returns>
|
||||||
public static GeoBox FromPoint(GeoPoint point)
|
public static GeoBox FromPoint(GeoPoint point)
|
||||||
{
|
{
|
||||||
return new GeoBox(point.Latitude, point.Longitude, point.Latitude, point.Longitude);
|
return new GeoBox(point.Latitude, point.Longitude, point.Latitude, point.Longitude);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Expands this box to include the specified point.
|
/// Expands this box to include the specified point.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="point">The point to include.</param>
|
/// <param name="point">The point to include.</param>
|
||||||
/// <returns>A new expanded bounding box.</returns>
|
/// <returns>A new expanded bounding box.</returns>
|
||||||
public GeoBox ExpandTo(GeoPoint point)
|
public GeoBox ExpandTo(GeoPoint point)
|
||||||
{
|
{
|
||||||
return new GeoBox(
|
return new GeoBox(
|
||||||
Math.Min(MinLat, point.Latitude),
|
Math.Min(MinLat, point.Latitude),
|
||||||
Math.Min(MinLon, point.Longitude),
|
Math.Min(MinLon, point.Longitude),
|
||||||
Math.Max(MaxLat, point.Latitude),
|
Math.Max(MaxLat, point.Latitude),
|
||||||
Math.Max(MaxLon, point.Longitude));
|
Math.Max(MaxLon, point.Longitude));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Expands this box to include the specified box.
|
/// Expands this box to include the specified box.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="other">The box to include.</param>
|
/// <param name="other">The box to include.</param>
|
||||||
/// <returns>A new expanded bounding box.</returns>
|
/// <returns>A new expanded bounding box.</returns>
|
||||||
public GeoBox ExpandTo(GeoBox other)
|
public GeoBox ExpandTo(GeoBox other)
|
||||||
{
|
{
|
||||||
return new GeoBox(
|
return new GeoBox(
|
||||||
Math.Min(MinLat, other.MinLat),
|
Math.Min(MinLat, other.MinLat),
|
||||||
Math.Min(MinLon, other.MinLon),
|
Math.Min(MinLon, other.MinLon),
|
||||||
Math.Max(MaxLat, other.MaxLat),
|
Math.Max(MaxLat, other.MaxLat),
|
||||||
Math.Max(MaxLon, other.MaxLon));
|
Math.Max(MaxLon, other.MaxLon));
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Gets the area of this bounding box.
|
|
||||||
/// </summary>
|
|
||||||
public double Area => Math.Max(0, MaxLat - MinLat) * Math.Max(0, MaxLon - MinLon);
|
|
||||||
}
|
|
||||||
@@ -1,27 +1,25 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
|
|
||||||
public struct InternalEntry
|
public struct InternalEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the separator key.
|
/// Gets or sets the separator key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexKey Key { get; set; }
|
public IndexKey Key { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the child page identifier.
|
/// Gets or sets the child page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageId { get; set; }
|
public uint PageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="InternalEntry"/> struct.
|
/// Initializes a new instance of the <see cref="InternalEntry" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The separator key.</param>
|
/// <param name="key">The separator key.</param>
|
||||||
/// <param name="pageId">The child page identifier.</param>
|
/// <param name="pageId">The child page identifier.</param>
|
||||||
public InternalEntry(IndexKey key, uint pageId)
|
public InternalEntry(IndexKey key, uint pageId)
|
||||||
{
|
{
|
||||||
Key = key;
|
Key = key;
|
||||||
PageId = pageId;
|
PageId = pageId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,73 +1,78 @@
|
|||||||
|
using System.Buffers;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
|
||||||
using System.Buffers;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// R-Tree Index implementation for Geospatial Indexing.
|
/// R-Tree Index implementation for Geospatial Indexing.
|
||||||
/// Uses Quadratic Split algorithm for simplicity and efficiency in paged storage.
|
/// Uses Quadratic Split algorithm for simplicity and efficiency in paged storage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal class RTreeIndex : IDisposable
|
internal class RTreeIndex : IDisposable
|
||||||
{
|
{
|
||||||
private readonly IIndexStorage _storage;
|
|
||||||
private readonly IndexOptions _options;
|
|
||||||
private uint _rootPageId;
|
|
||||||
private readonly object _lock = new();
|
private readonly object _lock = new();
|
||||||
|
private readonly IndexOptions _options;
|
||||||
private readonly int _pageSize;
|
private readonly int _pageSize;
|
||||||
|
private readonly IIndexStorage _storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="RTreeIndex"/> class.
|
/// Initializes a new instance of the <see cref="RTreeIndex" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="storage">The storage engine used for page operations.</param>
|
/// <param name="storage">The storage engine used for page operations.</param>
|
||||||
/// <param name="options">The index options.</param>
|
/// <param name="options">The index options.</param>
|
||||||
/// <param name="rootPageId">The root page identifier, or <c>0</c> to create a new root.</param>
|
/// <param name="rootPageId">The root page identifier, or <c>0</c> to create a new root.</param>
|
||||||
public RTreeIndex(IIndexStorage storage, IndexOptions options, uint rootPageId)
|
public RTreeIndex(IIndexStorage storage, IndexOptions options, uint rootPageId)
|
||||||
{
|
{
|
||||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||||
_options = options;
|
_options = options;
|
||||||
_rootPageId = rootPageId;
|
RootPageId = rootPageId;
|
||||||
_pageSize = _storage.PageSize;
|
_pageSize = _storage.PageSize;
|
||||||
|
|
||||||
if (_rootPageId == 0)
|
if (RootPageId == 0) InitializeNewIndex();
|
||||||
{
|
}
|
||||||
InitializeNewIndex();
|
|
||||||
}
|
/// <summary>
|
||||||
}
|
/// Gets the current root page identifier.
|
||||||
|
/// </summary>
|
||||||
/// <summary>
|
public uint RootPageId { get; private set; }
|
||||||
/// Gets the current root page identifier.
|
|
||||||
/// </summary>
|
/// <summary>
|
||||||
public uint RootPageId => _rootPageId;
|
/// Releases resources used by the index.
|
||||||
|
/// </summary>
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
private void InitializeNewIndex()
|
private void InitializeNewIndex()
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_rootPageId = _storage.AllocatePage();
|
RootPageId = _storage.AllocatePage();
|
||||||
SpatialPage.Initialize(buffer, _rootPageId, true, 0);
|
SpatialPage.Initialize(buffer, RootPageId, true, 0);
|
||||||
_storage.WritePageImmediate(_rootPageId, buffer);
|
_storage.WritePageImmediate(RootPageId, buffer);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Searches for document locations whose minimum bounding rectangles intersect the specified area.
|
/// Searches for document locations whose minimum bounding rectangles intersect the specified area.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="area">The area to search.</param>
|
/// <param name="area">The area to search.</param>
|
||||||
/// <param name="transaction">The optional transaction context.</param>
|
/// <param name="transaction">The optional transaction context.</param>
|
||||||
/// <returns>A sequence of matching document locations.</returns>
|
/// <returns>A sequence of matching document locations.</returns>
|
||||||
public IEnumerable<DocumentLocation> Search(GeoBox area, ITransaction? transaction = null)
|
public IEnumerable<DocumentLocation> Search(GeoBox area, ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
if (_rootPageId == 0) yield break;
|
if (RootPageId == 0) yield break;
|
||||||
|
|
||||||
var stack = new Stack<uint>();
|
var stack = new Stack<uint>();
|
||||||
stack.Push(_rootPageId);
|
stack.Push(RootPageId);
|
||||||
|
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
while (stack.Count > 0)
|
while (stack.Count > 0)
|
||||||
@@ -78,38 +83,37 @@ internal class RTreeIndex : IDisposable
|
|||||||
bool isLeaf = SpatialPage.GetIsLeaf(buffer);
|
bool isLeaf = SpatialPage.GetIsLeaf(buffer);
|
||||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||||
|
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
SpatialPage.ReadEntry(buffer, i, out var mbr, out var pointer);
|
SpatialPage.ReadEntry(buffer, i, out var mbr, out var pointer);
|
||||||
|
|
||||||
if (area.Intersects(mbr))
|
if (area.Intersects(mbr))
|
||||||
{
|
{
|
||||||
if (isLeaf)
|
if (isLeaf)
|
||||||
{
|
|
||||||
yield return pointer;
|
yield return pointer;
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
|
||||||
stack.Push(pointer.PageId);
|
stack.Push(pointer.PageId);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a bounding rectangle and document location into the index.
|
/// Inserts a bounding rectangle and document location into the index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="mbr">The minimum bounding rectangle to index.</param>
|
/// <param name="mbr">The minimum bounding rectangle to index.</param>
|
||||||
/// <param name="loc">The document location associated with the rectangle.</param>
|
/// <param name="loc">The document location associated with the rectangle.</param>
|
||||||
/// <param name="transaction">The optional transaction context.</param>
|
/// <param name="transaction">The optional transaction context.</param>
|
||||||
public void Insert(GeoBox mbr, DocumentLocation loc, ITransaction? transaction = null)
|
public void Insert(GeoBox mbr, DocumentLocation loc, ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
lock (_lock)
|
lock (_lock)
|
||||||
{
|
{
|
||||||
var leafPageId = ChooseLeaf(_rootPageId, mbr, transaction);
|
uint leafPageId = ChooseLeaf(RootPageId, mbr, transaction);
|
||||||
InsertIntoNode(leafPageId, mbr, loc, transaction);
|
InsertIntoNode(leafPageId, mbr, loc, transaction);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -117,7 +121,7 @@ internal class RTreeIndex : IDisposable
|
|||||||
private uint ChooseLeaf(uint rootId, GeoBox mbr, ITransaction? transaction)
|
private uint ChooseLeaf(uint rootId, GeoBox mbr, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
uint currentId = rootId;
|
uint currentId = rootId;
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
while (true)
|
while (true)
|
||||||
@@ -127,13 +131,13 @@ internal class RTreeIndex : IDisposable
|
|||||||
|
|
||||||
ushort count = SpatialPage.GetEntryCount(buffer);
|
ushort count = SpatialPage.GetEntryCount(buffer);
|
||||||
uint bestChild = 0;
|
uint bestChild = 0;
|
||||||
double minEnlargement = double.MaxValue;
|
var minEnlargement = double.MaxValue;
|
||||||
double minArea = double.MaxValue;
|
var minArea = double.MaxValue;
|
||||||
|
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
SpatialPage.ReadEntry(buffer, i, out var childMbr, out var pointer);
|
SpatialPage.ReadEntry(buffer, i, out var childMbr, out var pointer);
|
||||||
|
|
||||||
var expanded = childMbr.ExpandTo(mbr);
|
var expanded = childMbr.ExpandTo(mbr);
|
||||||
double enlargement = expanded.Area - childMbr.Area;
|
double enlargement = expanded.Area - childMbr.Area;
|
||||||
|
|
||||||
@@ -156,12 +160,15 @@ internal class RTreeIndex : IDisposable
|
|||||||
currentId = bestChild;
|
currentId = bestChild;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void InsertIntoNode(uint pageId, GeoBox mbr, DocumentLocation pointer, ITransaction? transaction)
|
private void InsertIntoNode(uint pageId, GeoBox mbr, DocumentLocation pointer, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||||
@@ -171,8 +178,8 @@ internal class RTreeIndex : IDisposable
|
|||||||
if (count < maxEntries)
|
if (count < maxEntries)
|
||||||
{
|
{
|
||||||
SpatialPage.WriteEntry(buffer, count, mbr, pointer);
|
SpatialPage.WriteEntry(buffer, count, mbr, pointer);
|
||||||
SpatialPage.SetEntryCount(buffer, (ushort)(count + 1));
|
SpatialPage.SetEntryCount(buffer, (ushort)(count + 1));
|
||||||
|
|
||||||
if (transaction != null)
|
if (transaction != null)
|
||||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||||
else
|
else
|
||||||
@@ -186,17 +193,20 @@ internal class RTreeIndex : IDisposable
|
|||||||
SplitNode(pageId, mbr, pointer, transaction);
|
SplitNode(pageId, mbr, pointer, transaction);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void UpdateMBRUpwards(uint pageId, ITransaction? transaction)
|
private void UpdateMBRUpwards(uint pageId, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
var parentBuffer = RentPageBuffer();
|
byte[] parentBuffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
uint currentId = pageId;
|
uint currentId = pageId;
|
||||||
while (currentId != _rootPageId)
|
while (currentId != RootPageId)
|
||||||
{
|
{
|
||||||
_storage.ReadPage(currentId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(currentId, transaction?.TransactionId, buffer);
|
||||||
var currentMbr = SpatialPage.CalculateMBR(buffer);
|
var currentMbr = SpatialPage.CalculateMBR(buffer);
|
||||||
@@ -206,9 +216,9 @@ internal class RTreeIndex : IDisposable
|
|||||||
|
|
||||||
_storage.ReadPage(parentId, transaction?.TransactionId, parentBuffer);
|
_storage.ReadPage(parentId, transaction?.TransactionId, parentBuffer);
|
||||||
ushort count = SpatialPage.GetEntryCount(parentBuffer);
|
ushort count = SpatialPage.GetEntryCount(parentBuffer);
|
||||||
bool changed = false;
|
var changed = false;
|
||||||
|
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
SpatialPage.ReadEntry(parentBuffer, i, out var mbr, out var pointer);
|
SpatialPage.ReadEntry(parentBuffer, i, out var mbr, out var pointer);
|
||||||
if (pointer.PageId == currentId)
|
if (pointer.PageId == currentId)
|
||||||
@@ -218,6 +228,7 @@ internal class RTreeIndex : IDisposable
|
|||||||
SpatialPage.WriteEntry(parentBuffer, i, currentMbr, pointer);
|
SpatialPage.WriteEntry(parentBuffer, i, currentMbr, pointer);
|
||||||
changed = true;
|
changed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -232,17 +243,17 @@ internal class RTreeIndex : IDisposable
|
|||||||
currentId = parentId;
|
currentId = parentId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
ReturnPageBuffer(buffer);
|
ReturnPageBuffer(buffer);
|
||||||
ReturnPageBuffer(parentBuffer);
|
ReturnPageBuffer(parentBuffer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void SplitNode(uint pageId, GeoBox newMbr, DocumentLocation newPointer, ITransaction? transaction)
|
private void SplitNode(uint pageId, GeoBox newMbr, DocumentLocation newPointer, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
var newBuffer = RentPageBuffer();
|
byte[] newBuffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||||
@@ -253,11 +264,12 @@ internal class RTreeIndex : IDisposable
|
|||||||
|
|
||||||
// Collect all entries including the new one
|
// Collect all entries including the new one
|
||||||
var entries = new List<(GeoBox Mbr, DocumentLocation Pointer)>();
|
var entries = new List<(GeoBox Mbr, DocumentLocation Pointer)>();
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
SpatialPage.ReadEntry(buffer, i, out var m, out var p);
|
SpatialPage.ReadEntry(buffer, i, out var m, out var p);
|
||||||
entries.Add((m, p));
|
entries.Add((m, p));
|
||||||
}
|
}
|
||||||
|
|
||||||
entries.Add((newMbr, newPointer));
|
entries.Add((newMbr, newPointer));
|
||||||
|
|
||||||
// Pick Seeds
|
// Pick Seeds
|
||||||
@@ -277,8 +289,8 @@ internal class RTreeIndex : IDisposable
|
|||||||
SpatialPage.WriteEntry(newBuffer, 0, seed2.Mbr, seed2.Pointer);
|
SpatialPage.WriteEntry(newBuffer, 0, seed2.Mbr, seed2.Pointer);
|
||||||
SpatialPage.SetEntryCount(newBuffer, 1);
|
SpatialPage.SetEntryCount(newBuffer, 1);
|
||||||
|
|
||||||
GeoBox mbr1 = seed1.Mbr;
|
var mbr1 = seed1.Mbr;
|
||||||
GeoBox mbr2 = seed2.Mbr;
|
var mbr2 = seed2.Mbr;
|
||||||
|
|
||||||
// Distribute remaining entries
|
// Distribute remaining entries
|
||||||
while (entries.Count > 0)
|
while (entries.Count > 0)
|
||||||
@@ -320,23 +332,23 @@ internal class RTreeIndex : IDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Propagate split upwards
|
// Propagate split upwards
|
||||||
if (pageId == _rootPageId)
|
if (pageId == RootPageId)
|
||||||
{
|
{
|
||||||
// New Root
|
// New Root
|
||||||
uint newRootId = _storage.AllocatePage();
|
uint newRootId = _storage.AllocatePage();
|
||||||
SpatialPage.Initialize(buffer, newRootId, false, (byte)(level + 1));
|
SpatialPage.Initialize(buffer, newRootId, false, (byte)(level + 1));
|
||||||
SpatialPage.WriteEntry(buffer, 0, mbr1, new DocumentLocation(pageId, 0));
|
SpatialPage.WriteEntry(buffer, 0, mbr1, new DocumentLocation(pageId, 0));
|
||||||
SpatialPage.WriteEntry(buffer, 1, mbr2, new DocumentLocation(newPageId, 0));
|
SpatialPage.WriteEntry(buffer, 1, mbr2, new DocumentLocation(newPageId, 0));
|
||||||
SpatialPage.SetEntryCount(buffer, 2);
|
SpatialPage.SetEntryCount(buffer, 2);
|
||||||
|
|
||||||
if (transaction != null)
|
if (transaction != null)
|
||||||
_storage.WritePage(newRootId, transaction.TransactionId, buffer);
|
_storage.WritePage(newRootId, transaction.TransactionId, buffer);
|
||||||
else
|
else
|
||||||
_storage.WritePageImmediate(newRootId, buffer);
|
_storage.WritePageImmediate(newRootId, buffer);
|
||||||
|
|
||||||
_rootPageId = newRootId;
|
RootPageId = newRootId;
|
||||||
|
|
||||||
// Update parent pointers
|
// Update parent pointers
|
||||||
UpdateParentPointer(pageId, newRootId, transaction);
|
UpdateParentPointer(pageId, newRootId, transaction);
|
||||||
UpdateParentPointer(newPageId, newRootId, transaction);
|
UpdateParentPointer(newPageId, newRootId, transaction);
|
||||||
}
|
}
|
||||||
@@ -347,16 +359,16 @@ internal class RTreeIndex : IDisposable
|
|||||||
UpdateMBRUpwards(pageId, transaction);
|
UpdateMBRUpwards(pageId, transaction);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
ReturnPageBuffer(buffer);
|
ReturnPageBuffer(buffer);
|
||||||
ReturnPageBuffer(newBuffer);
|
ReturnPageBuffer(newBuffer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void UpdateParentPointer(uint pageId, uint parentId, ITransaction? transaction)
|
private void UpdateParentPointer(uint pageId, uint parentId, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||||
@@ -366,27 +378,29 @@ internal class RTreeIndex : IDisposable
|
|||||||
else
|
else
|
||||||
_storage.WritePageImmediate(pageId, buffer);
|
_storage.WritePageImmediate(pageId, buffer);
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void PickSeeds(List<(GeoBox Mbr, DocumentLocation Pointer)> entries, out (GeoBox Mbr, DocumentLocation Pointer) s1, out (GeoBox Mbr, DocumentLocation Pointer) s2)
|
private void PickSeeds(List<(GeoBox Mbr, DocumentLocation Pointer)> entries,
|
||||||
|
out (GeoBox Mbr, DocumentLocation Pointer) s1, out (GeoBox Mbr, DocumentLocation Pointer) s2)
|
||||||
{
|
{
|
||||||
double maxWaste = double.MinValue;
|
var maxWaste = double.MinValue;
|
||||||
s1 = entries[0];
|
s1 = entries[0];
|
||||||
s2 = entries[1];
|
s2 = entries[1];
|
||||||
|
|
||||||
for (int i = 0; i < entries.Count; i++)
|
for (var i = 0; i < entries.Count; i++)
|
||||||
|
for (int j = i + 1; j < entries.Count; j++)
|
||||||
{
|
{
|
||||||
for (int j = i + 1; j < entries.Count; j++)
|
var combined = entries[i].Mbr.ExpandTo(entries[j].Mbr);
|
||||||
|
double waste = combined.Area - entries[i].Mbr.Area - entries[j].Mbr.Area;
|
||||||
|
if (waste > maxWaste)
|
||||||
{
|
{
|
||||||
var combined = entries[i].Mbr.ExpandTo(entries[j].Mbr);
|
maxWaste = waste;
|
||||||
double waste = combined.Area - entries[i].Mbr.Area - entries[j].Mbr.Area;
|
s1 = entries[i];
|
||||||
if (waste > maxWaste)
|
s2 = entries[j];
|
||||||
{
|
|
||||||
maxWaste = waste;
|
|
||||||
s1 = entries[i];
|
|
||||||
s2 = entries[j];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -400,11 +414,4 @@ internal class RTreeIndex : IDisposable
|
|||||||
{
|
{
|
||||||
ArrayPool<byte>.Shared.Return(buffer);
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Releases resources used by the index.
|
|
||||||
/// </summary>
|
|
||||||
public void Dispose()
|
|
||||||
{
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,22 +1,25 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
public static class SpatialMath
|
public static class SpatialMath
|
||||||
{
|
{
|
||||||
private const double EarthRadiusKm = 6371.0;
|
private const double EarthRadiusKm = 6371.0;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Calculates distance between two points on Earth using Haversine formula.
|
/// Calculates distance between two points on Earth using Haversine formula.
|
||||||
/// Result in kilometers.
|
/// Result in kilometers.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="p1">The first point.</param>
|
/// <param name="p1">The first point.</param>
|
||||||
/// <param name="p2">The second point.</param>
|
/// <param name="p2">The second point.</param>
|
||||||
/// <returns>The distance in kilometers.</returns>
|
/// <returns>The distance in kilometers.</returns>
|
||||||
internal static double DistanceKm(GeoPoint p1, GeoPoint p2) => DistanceKm(p1.Latitude, p1.Longitude, p2.Latitude, p2.Longitude);
|
internal static double DistanceKm(GeoPoint p1, GeoPoint p2)
|
||||||
|
{
|
||||||
|
return DistanceKm(p1.Latitude, p1.Longitude, p2.Latitude, p2.Longitude);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Calculates distance between two coordinates on Earth using Haversine formula.
|
/// Calculates distance between two coordinates on Earth using Haversine formula.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="lat1">The latitude of the first point.</param>
|
/// <param name="lat1">The latitude of the first point.</param>
|
||||||
/// <param name="lon1">The longitude of the first point.</param>
|
/// <param name="lon1">The longitude of the first point.</param>
|
||||||
@@ -27,34 +30,40 @@ public static class SpatialMath
|
|||||||
{
|
{
|
||||||
double dLat = ToRadians(lat2 - lat1);
|
double dLat = ToRadians(lat2 - lat1);
|
||||||
double dLon = ToRadians(lon2 - lon1);
|
double dLon = ToRadians(lon2 - lon1);
|
||||||
|
|
||||||
double a = Math.Sin(dLat / 2) * Math.Sin(dLat / 2) +
|
double a = Math.Sin(dLat / 2) * Math.Sin(dLat / 2) +
|
||||||
Math.Cos(ToRadians(lat1)) * Math.Cos(ToRadians(lat2)) *
|
Math.Cos(ToRadians(lat1)) * Math.Cos(ToRadians(lat2)) *
|
||||||
Math.Sin(dLon / 2) * Math.Sin(dLon / 2);
|
Math.Sin(dLon / 2) * Math.Sin(dLon / 2);
|
||||||
|
|
||||||
double c = 2 * Math.Atan2(Math.Sqrt(a), Math.Sqrt(1 - a));
|
double c = 2 * Math.Atan2(Math.Sqrt(a), Math.Sqrt(1 - a));
|
||||||
return EarthRadiusKm * c;
|
return EarthRadiusKm * c;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a Bounding Box (MBR) centered at a point with a given radius in km.
|
/// Creates a Bounding Box (MBR) centered at a point with a given radius in km.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="center">The center point.</param>
|
/// <param name="center">The center point.</param>
|
||||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||||
/// <returns>The bounding box.</returns>
|
/// <returns>The bounding box.</returns>
|
||||||
internal static GeoBox BoundingBox(GeoPoint center, double radiusKm) => BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
internal static GeoBox BoundingBox(GeoPoint center, double radiusKm)
|
||||||
|
{
|
||||||
|
return BoundingBox(center.Latitude, center.Longitude, radiusKm);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a bounding box from a coordinate and radius.
|
/// Creates a bounding box from a coordinate and radius.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="lat">The center latitude.</param>
|
/// <param name="lat">The center latitude.</param>
|
||||||
/// <param name="lon">The center longitude.</param>
|
/// <param name="lon">The center longitude.</param>
|
||||||
/// <param name="radiusKm">The radius in kilometers.</param>
|
/// <param name="radiusKm">The radius in kilometers.</param>
|
||||||
/// <returns>The bounding box.</returns>
|
/// <returns>The bounding box.</returns>
|
||||||
internal static GeoBox InternalBoundingBox(double lat, double lon, double radiusKm) => BoundingBox(lat, lon, radiusKm);
|
internal static GeoBox InternalBoundingBox(double lat, double lon, double radiusKm)
|
||||||
|
{
|
||||||
|
return BoundingBox(lat, lon, radiusKm);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a bounding box centered at a coordinate with a given radius in kilometers.
|
/// Creates a bounding box centered at a coordinate with a given radius in kilometers.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="lat">The center latitude.</param>
|
/// <param name="lat">The center latitude.</param>
|
||||||
/// <param name="lon">The center longitude.</param>
|
/// <param name="lon">The center longitude.</param>
|
||||||
@@ -64,14 +73,21 @@ public static class SpatialMath
|
|||||||
{
|
{
|
||||||
double dLat = ToDegrees(radiusKm / EarthRadiusKm);
|
double dLat = ToDegrees(radiusKm / EarthRadiusKm);
|
||||||
double dLon = ToDegrees(radiusKm / (EarthRadiusKm * Math.Cos(ToRadians(lat))));
|
double dLon = ToDegrees(radiusKm / (EarthRadiusKm * Math.Cos(ToRadians(lat))));
|
||||||
|
|
||||||
return new GeoBox(
|
return new GeoBox(
|
||||||
lat - dLat,
|
lat - dLat,
|
||||||
lon - dLon,
|
lon - dLon,
|
||||||
lat + dLat,
|
lat + dLat,
|
||||||
lon + dLon);
|
lon + dLon);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static double ToRadians(double degrees) => degrees * Math.PI / 180.0;
|
private static double ToRadians(double degrees)
|
||||||
private static double ToDegrees(double radians) => radians * 180.0 / Math.PI;
|
{
|
||||||
}
|
return degrees * Math.PI / 180.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static double ToDegrees(double radians)
|
||||||
|
{
|
||||||
|
return radians * 180.0 / Math.PI;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,23 +1,21 @@
|
|||||||
using System.Runtime.Intrinsics;
|
|
||||||
using System.Runtime.Intrinsics.X86;
|
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
using System.Numerics;
|
using System.Numerics;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Optimized vector math utilities using SIMD if available.
|
/// Optimized vector math utilities using SIMD if available.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static class VectorMath
|
public static class VectorMath
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes vector distance according to the selected metric.
|
/// Computes vector distance according to the selected metric.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="v1">The first vector.</param>
|
/// <param name="v1">The first vector.</param>
|
||||||
/// <param name="v2">The second vector.</param>
|
/// <param name="v2">The second vector.</param>
|
||||||
/// <param name="metric">The metric used to compute distance.</param>
|
/// <param name="metric">The metric used to compute distance.</param>
|
||||||
/// <returns>The distance value for the selected metric.</returns>
|
/// <returns>The distance value for the selected metric.</returns>
|
||||||
public static float Distance(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2, VectorMetric metric)
|
public static float Distance(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2, VectorMetric metric)
|
||||||
{
|
{
|
||||||
return metric switch
|
return metric switch
|
||||||
{
|
{
|
||||||
@@ -28,13 +26,13 @@ public static class VectorMath
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes cosine similarity between two vectors.
|
/// Computes cosine similarity between two vectors.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="v1">The first vector.</param>
|
/// <param name="v1">The first vector.</param>
|
||||||
/// <param name="v2">The second vector.</param>
|
/// <param name="v2">The second vector.</param>
|
||||||
/// <returns>The cosine similarity in the range [-1, 1].</returns>
|
/// <returns>The cosine similarity in the range [-1, 1].</returns>
|
||||||
public static float CosineSimilarity(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
public static float CosineSimilarity(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||||
{
|
{
|
||||||
float dot = DotProduct(v1, v2);
|
float dot = DotProduct(v1, v2);
|
||||||
float mag1 = DotProduct(v1, v1);
|
float mag1 = DotProduct(v1, v1);
|
||||||
@@ -44,19 +42,19 @@ public static class VectorMath
|
|||||||
return dot / (MathF.Sqrt(mag1) * MathF.Sqrt(mag2));
|
return dot / (MathF.Sqrt(mag1) * MathF.Sqrt(mag2));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes the dot product of two vectors.
|
/// Computes the dot product of two vectors.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="v1">The first vector.</param>
|
/// <param name="v1">The first vector.</param>
|
||||||
/// <param name="v2">The second vector.</param>
|
/// <param name="v2">The second vector.</param>
|
||||||
/// <returns>The dot product value.</returns>
|
/// <returns>The dot product value.</returns>
|
||||||
public static float DotProduct(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
public static float DotProduct(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||||
{
|
{
|
||||||
if (v1.Length != v2.Length)
|
if (v1.Length != v2.Length)
|
||||||
throw new ArgumentException("Vectors must have same length");
|
throw new ArgumentException("Vectors must have same length");
|
||||||
|
|
||||||
float dot = 0;
|
float dot = 0;
|
||||||
int i = 0;
|
var i = 0;
|
||||||
|
|
||||||
// SIMD Optimization for .NET
|
// SIMD Optimization for .NET
|
||||||
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
||||||
@@ -65,37 +63,31 @@ public static class VectorMath
|
|||||||
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
||||||
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
||||||
|
|
||||||
foreach (var chunk in Enumerable.Range(0, v1Span.Length))
|
foreach (int chunk in Enumerable.Range(0, v1Span.Length)) vDot += v1Span[chunk] * v2Span[chunk];
|
||||||
{
|
|
||||||
vDot += v1Span[chunk] * v2Span[chunk];
|
|
||||||
}
|
|
||||||
|
|
||||||
dot = Vector.Dot(vDot, Vector<float>.One);
|
dot = Vector.Dot(vDot, Vector<float>.One);
|
||||||
i = v1Span.Length * Vector<float>.Count;
|
i = v1Span.Length * Vector<float>.Count;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remaining elements
|
// Remaining elements
|
||||||
for (; i < v1.Length; i++)
|
for (; i < v1.Length; i++) dot += v1[i] * v2[i];
|
||||||
{
|
|
||||||
dot += v1[i] * v2[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
return dot;
|
return dot;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Computes squared Euclidean distance between two vectors.
|
/// Computes squared Euclidean distance between two vectors.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="v1">The first vector.</param>
|
/// <param name="v1">The first vector.</param>
|
||||||
/// <param name="v2">The second vector.</param>
|
/// <param name="v2">The second vector.</param>
|
||||||
/// <returns>The squared Euclidean distance.</returns>
|
/// <returns>The squared Euclidean distance.</returns>
|
||||||
public static float EuclideanDistanceSquared(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
public static float EuclideanDistanceSquared(ReadOnlySpan<float> v1, ReadOnlySpan<float> v2)
|
||||||
{
|
{
|
||||||
if (v1.Length != v2.Length)
|
if (v1.Length != v2.Length)
|
||||||
throw new ArgumentException("Vectors must have same length");
|
throw new ArgumentException("Vectors must have same length");
|
||||||
|
|
||||||
float dist = 0;
|
float dist = 0;
|
||||||
int i = 0;
|
var i = 0;
|
||||||
|
|
||||||
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
if (Vector.IsHardwareAccelerated && v1.Length >= Vector<float>.Count)
|
||||||
{
|
{
|
||||||
@@ -103,7 +95,7 @@ public static class VectorMath
|
|||||||
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
var v1Span = MemoryMarshal.Cast<float, Vector<float>>(v1);
|
||||||
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
var v2Span = MemoryMarshal.Cast<float, Vector<float>>(v2);
|
||||||
|
|
||||||
foreach (var chunk in Enumerable.Range(0, v1Span.Length))
|
foreach (int chunk in Enumerable.Range(0, v1Span.Length))
|
||||||
{
|
{
|
||||||
var diff = v1Span[chunk] - v2Span[chunk];
|
var diff = v1Span[chunk] - v2Span[chunk];
|
||||||
vDist += diff * diff;
|
vDist += diff * diff;
|
||||||
@@ -121,4 +113,4 @@ public static class VectorMath
|
|||||||
|
|
||||||
return dist;
|
return dist;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,28 +3,34 @@ namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
|||||||
public static class VectorSearchExtensions
|
public static class VectorSearchExtensions
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a similarity search on a vector property.
|
/// Performs a similarity search on a vector property.
|
||||||
/// This method is a marker for the LINQ query provider and is optimized using HNSW indexes if available.
|
/// This method is a marker for the LINQ query provider and is optimized using HNSW indexes if available.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vector">The vector property of the entity.</param>
|
/// <param name="vector">The vector property of the entity.</param>
|
||||||
/// <param name="query">The query vector to compare against.</param>
|
/// <param name="query">The query vector to compare against.</param>
|
||||||
/// <param name="k">Number of nearest neighbors to return.</param>
|
/// <param name="k">Number of nearest neighbors to return.</param>
|
||||||
/// <returns>True if the document is part of the top-k results (always returns true when evaluated in memory for compilation purposes).</returns>
|
/// <returns>
|
||||||
|
/// True if the document is part of the top-k results (always returns true when evaluated in memory for
|
||||||
|
/// compilation purposes).
|
||||||
|
/// </returns>
|
||||||
public static bool VectorSearch(this float[] vector, float[] query, int k)
|
public static bool VectorSearch(this float[] vector, float[] query, int k)
|
||||||
{
|
{
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a similarity search on a collection of vector properties.
|
/// Performs a similarity search on a collection of vector properties.
|
||||||
/// Used for entities with multiple vectors per document.
|
/// Used for entities with multiple vectors per document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vectors">The vector collection of the entity.</param>
|
/// <param name="vectors">The vector collection of the entity.</param>
|
||||||
/// <param name="query">The query vector to compare against.</param>
|
/// <param name="query">The query vector to compare against.</param>
|
||||||
/// <param name="k">Number of nearest neighbors to return.</param>
|
/// <param name="k">Number of nearest neighbors to return.</param>
|
||||||
/// <returns>True if the document is part of the top-k results (always returns true when evaluated in memory for compilation purposes).</returns>
|
/// <returns>
|
||||||
public static bool VectorSearch(this IEnumerable<float[]> vectors, float[] query, int k)
|
/// True if the document is part of the top-k results (always returns true when evaluated in memory for
|
||||||
{
|
/// compilation purposes).
|
||||||
return true;
|
/// </returns>
|
||||||
}
|
public static bool VectorSearch(this IEnumerable<float[]> vectors, float[] query, int k)
|
||||||
}
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,87 +1,85 @@
|
|||||||
|
using System.Buffers;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
namespace ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// HNSW (Hierarchical Navigable Small World) index implementation.
|
/// HNSW (Hierarchical Navigable Small World) index implementation.
|
||||||
/// Handles multi-vector indexing and similarity searches.
|
/// Handles multi-vector indexing and similarity searches.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class VectorSearchIndex
|
public sealed class VectorSearchIndex
|
||||||
{
|
{
|
||||||
private struct NodeReference
|
|
||||||
{
|
|
||||||
public uint PageId;
|
|
||||||
public int NodeIndex;
|
|
||||||
public int MaxLevel;
|
|
||||||
}
|
|
||||||
|
|
||||||
private readonly IIndexStorage _storage;
|
|
||||||
private readonly IndexOptions _options;
|
private readonly IndexOptions _options;
|
||||||
private uint _rootPageId;
|
|
||||||
private readonly Random _random = new(42);
|
private readonly Random _random = new(42);
|
||||||
|
|
||||||
/// <summary>
|
private readonly IIndexStorage _storage;
|
||||||
/// Initializes a new vector search index.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="storage">The storage engine used by the index.</param>
|
|
||||||
/// <param name="options">Index configuration options.</param>
|
|
||||||
/// <param name="rootPageId">Optional existing root page identifier.</param>
|
|
||||||
public VectorSearchIndex(StorageEngine storage, IndexOptions options, uint rootPageId = 0)
|
|
||||||
: this((IStorageEngine)storage, options, rootPageId)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new vector search index.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="storage">The index storage abstraction used by the index.</param>
|
|
||||||
/// <param name="options">Index configuration options.</param>
|
|
||||||
/// <param name="rootPageId">Optional existing root page identifier.</param>
|
|
||||||
internal VectorSearchIndex(IIndexStorage storage, IndexOptions options, uint rootPageId = 0)
|
|
||||||
{
|
|
||||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
|
||||||
_options = options;
|
|
||||||
_rootPageId = rootPageId;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the root page identifier of the index.
|
/// Initializes a new vector search index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint RootPageId => _rootPageId;
|
/// <param name="storage">The storage engine used by the index.</param>
|
||||||
|
/// <param name="options">Index configuration options.</param>
|
||||||
|
/// <param name="rootPageId">Optional existing root page identifier.</param>
|
||||||
|
public VectorSearchIndex(StorageEngine storage, IndexOptions options, uint rootPageId = 0)
|
||||||
|
: this((IStorageEngine)storage, options, rootPageId)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a vector and its document location into the index.
|
/// Initializes a new vector search index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="vector">The vector values to index.</param>
|
/// <param name="storage">The index storage abstraction used by the index.</param>
|
||||||
/// <param name="docLocation">The document location associated with the vector.</param>
|
/// <param name="options">Index configuration options.</param>
|
||||||
/// <param name="transaction">Optional transaction context.</param>
|
/// <param name="rootPageId">Optional existing root page identifier.</param>
|
||||||
public void Insert(float[] vector, DocumentLocation docLocation, ITransaction? transaction = null)
|
internal VectorSearchIndex(IIndexStorage storage, IndexOptions options, uint rootPageId = 0)
|
||||||
|
{
|
||||||
|
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||||
|
_options = options;
|
||||||
|
RootPageId = rootPageId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the root page identifier of the index.
|
||||||
|
/// </summary>
|
||||||
|
public uint RootPageId { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Inserts a vector and its document location into the index.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="vector">The vector values to index.</param>
|
||||||
|
/// <param name="docLocation">The document location associated with the vector.</param>
|
||||||
|
/// <param name="transaction">Optional transaction context.</param>
|
||||||
|
public void Insert(float[] vector, DocumentLocation docLocation, ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
if (vector.Length != _options.Dimensions)
|
if (vector.Length != _options.Dimensions)
|
||||||
throw new ArgumentException($"Vector dimension mismatch. Expected {_options.Dimensions}, got {vector.Length}");
|
throw new ArgumentException(
|
||||||
|
$"Vector dimension mismatch. Expected {_options.Dimensions}, got {vector.Length}");
|
||||||
|
|
||||||
// 1. Determine level for new node
|
// 1. Determine level for new node
|
||||||
int targetLevel = GetRandomLevel();
|
int targetLevel = GetRandomLevel();
|
||||||
|
|
||||||
// 2. If index is empty, create first page and first node
|
// 2. If index is empty, create first page and first node
|
||||||
if (_rootPageId == 0)
|
if (RootPageId == 0)
|
||||||
{
|
{
|
||||||
_rootPageId = CreateNewPage(transaction);
|
RootPageId = CreateNewPage(transaction);
|
||||||
var pageBuffer = RentPageBuffer();
|
byte[] pageBuffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(_rootPageId, transaction?.TransactionId, pageBuffer);
|
_storage.ReadPage(RootPageId, transaction?.TransactionId, pageBuffer);
|
||||||
VectorPage.WriteNode(pageBuffer, 0, docLocation, targetLevel, vector, _options.Dimensions);
|
VectorPage.WriteNode(pageBuffer, 0, docLocation, targetLevel, vector, _options.Dimensions);
|
||||||
VectorPage.IncrementNodeCount(pageBuffer); // Helper needs to be added or handled
|
VectorPage.IncrementNodeCount(pageBuffer); // Helper needs to be added or handled
|
||||||
|
|
||||||
if (transaction != null)
|
if (transaction != null)
|
||||||
_storage.WritePage(_rootPageId, transaction.TransactionId, pageBuffer);
|
_storage.WritePage(RootPageId, transaction.TransactionId, pageBuffer);
|
||||||
else
|
else
|
||||||
_storage.WritePageImmediate(_rootPageId, pageBuffer);
|
_storage.WritePageImmediate(RootPageId, pageBuffer);
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(pageBuffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(pageBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,9 +90,7 @@ public sealed class VectorSearchIndex
|
|||||||
|
|
||||||
// 4. Greedy search down to targetLevel+1
|
// 4. Greedy search down to targetLevel+1
|
||||||
for (int l = entryPoint.MaxLevel; l > targetLevel; l--)
|
for (int l = entryPoint.MaxLevel; l > targetLevel; l--)
|
||||||
{
|
|
||||||
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
||||||
}
|
|
||||||
|
|
||||||
// 5. Create the new node
|
// 5. Create the new node
|
||||||
var newNode = AllocateNode(vector, docLocation, targetLevel, transaction);
|
var newNode = AllocateNode(vector, docLocation, targetLevel, transaction);
|
||||||
@@ -103,25 +99,20 @@ public sealed class VectorSearchIndex
|
|||||||
for (int l = Math.Min(targetLevel, entryPoint.MaxLevel); l >= 0; l--)
|
for (int l = Math.Min(targetLevel, entryPoint.MaxLevel); l >= 0; l--)
|
||||||
{
|
{
|
||||||
var neighbors = SearchLayer(currentPoint, vector, _options.EfConstruction, l, transaction);
|
var neighbors = SearchLayer(currentPoint, vector, _options.EfConstruction, l, transaction);
|
||||||
var selectedNeighbors = SelectNeighbors(neighbors, vector, _options.M, l, transaction);
|
var selectedNeighbors = SelectNeighbors(neighbors, vector, _options.M, l, transaction);
|
||||||
|
|
||||||
foreach (var neighbor in selectedNeighbors)
|
foreach (var neighbor in selectedNeighbors) AddBidirectionalLink(newNode, neighbor, l, transaction);
|
||||||
{
|
|
||||||
AddBidirectionalLink(newNode, neighbor, l, transaction);
|
// Move currentPoint down for next level if available
|
||||||
}
|
|
||||||
|
|
||||||
// Move currentPoint down for next level if available
|
|
||||||
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
currentPoint = GreedySearch(currentPoint, vector, l, transaction);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 7. Update entry point if new node is higher
|
// 7. Update entry point if new node is higher
|
||||||
if (targetLevel > entryPoint.MaxLevel)
|
if (targetLevel > entryPoint.MaxLevel) UpdateEntryPoint(newNode, transaction);
|
||||||
{
|
|
||||||
UpdateEntryPoint(newNode, transaction);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<NodeReference> SelectNeighbors(IEnumerable<NodeReference> candidates, float[] query, int m, int level, ITransaction? transaction)
|
private IEnumerable<NodeReference> SelectNeighbors(IEnumerable<NodeReference> candidates, float[] query, int m,
|
||||||
|
int level, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
// Simple heuristic: just take top M nearest.
|
// Simple heuristic: just take top M nearest.
|
||||||
// HNSW Paper suggests more complex heuristic to maintain connectivity diversity.
|
// HNSW Paper suggests more complex heuristic to maintain connectivity diversity.
|
||||||
@@ -136,20 +127,20 @@ public sealed class VectorSearchIndex
|
|||||||
|
|
||||||
private void Link(NodeReference from, NodeReference to, int level, ITransaction? transaction)
|
private void Link(NodeReference from, NodeReference to, int level, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(from.PageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(from.PageId, transaction?.TransactionId, buffer);
|
||||||
var links = VectorPage.GetLinksSpan(buffer, from.NodeIndex, level, _options.Dimensions, _options.M);
|
var links = VectorPage.GetLinksSpan(buffer, from.NodeIndex, level, _options.Dimensions, _options.M);
|
||||||
|
|
||||||
// Find first empty slot (PageId == 0)
|
// Find first empty slot (PageId == 0)
|
||||||
for (int i = 0; i < links.Length; i += 6)
|
for (var i = 0; i < links.Length; i += 6)
|
||||||
{
|
{
|
||||||
var existing = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
var existing = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
||||||
if (existing.PageId == 0)
|
if (existing.PageId == 0)
|
||||||
{
|
{
|
||||||
new DocumentLocation(to.PageId, (ushort)to.NodeIndex).WriteTo(links.Slice(i, 6));
|
new DocumentLocation(to.PageId, (ushort)to.NodeIndex).WriteTo(links.Slice(i, 6));
|
||||||
|
|
||||||
if (transaction != null)
|
if (transaction != null)
|
||||||
_storage.WritePage(from.PageId, transaction.TransactionId, buffer);
|
_storage.WritePage(from.PageId, transaction.TransactionId, buffer);
|
||||||
else
|
else
|
||||||
@@ -160,7 +151,10 @@ public sealed class VectorSearchIndex
|
|||||||
// If full, we should technically prune or redistribute links as per HNSW paper.
|
// If full, we should technically prune or redistribute links as per HNSW paper.
|
||||||
// For now, we assume M is large enough or we skip (limited connectivity).
|
// For now, we assume M is large enough or we skip (limited connectivity).
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private NodeReference AllocateNode(float[] vector, DocumentLocation docLoc, int level, ITransaction? transaction)
|
private NodeReference AllocateNode(float[] vector, DocumentLocation docLoc, int level, ITransaction? transaction)
|
||||||
@@ -168,24 +162,27 @@ public sealed class VectorSearchIndex
|
|||||||
// Find a page with space or create new
|
// Find a page with space or create new
|
||||||
// For simplicity, we search for a page with available slots or append to a new one.
|
// For simplicity, we search for a page with available slots or append to a new one.
|
||||||
// Implementation omitted for brevity but required for full persistence.
|
// Implementation omitted for brevity but required for full persistence.
|
||||||
uint pageId = _rootPageId; // Placeholder: need allocation strategy
|
uint pageId = RootPageId; // Placeholder: need allocation strategy
|
||||||
int index = 0;
|
var index = 0;
|
||||||
|
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(pageId, transaction?.TransactionId, buffer);
|
||||||
index = VectorPage.GetNodeCount(buffer);
|
index = VectorPage.GetNodeCount(buffer);
|
||||||
VectorPage.WriteNode(buffer, index, docLoc, level, vector, _options.Dimensions);
|
VectorPage.WriteNode(buffer, index, docLoc, level, vector, _options.Dimensions);
|
||||||
VectorPage.IncrementNodeCount(buffer);
|
VectorPage.IncrementNodeCount(buffer);
|
||||||
|
|
||||||
if (transaction != null)
|
if (transaction != null)
|
||||||
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
_storage.WritePage(pageId, transaction.TransactionId, buffer);
|
||||||
else
|
else
|
||||||
_storage.WritePageImmediate(pageId, buffer);
|
_storage.WritePageImmediate(pageId, buffer);
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
return new NodeReference { PageId = pageId, NodeIndex = index, MaxLevel = level };
|
return new NodeReference { PageId = pageId, NodeIndex = index, MaxLevel = level };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,7 +194,7 @@ public sealed class VectorSearchIndex
|
|||||||
|
|
||||||
private NodeReference GreedySearch(NodeReference entryPoint, float[] query, int level, ITransaction? transaction)
|
private NodeReference GreedySearch(NodeReference entryPoint, float[] query, int level, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
bool changed = true;
|
var changed = true;
|
||||||
var current = entryPoint;
|
var current = entryPoint;
|
||||||
float currentDist = VectorMath.Distance(query, LoadVector(current, transaction), _options.Metric);
|
float currentDist = VectorMath.Distance(query, LoadVector(current, transaction), _options.Metric);
|
||||||
|
|
||||||
@@ -215,10 +212,12 @@ public sealed class VectorSearchIndex
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return current;
|
return current;
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<NodeReference> SearchLayer(NodeReference entryPoint, float[] query, int ef, int level, ITransaction? transaction)
|
private IEnumerable<NodeReference> SearchLayer(NodeReference entryPoint, float[] query, int ef, int level,
|
||||||
|
ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var visited = new HashSet<NodeReference>();
|
var visited = new HashSet<NodeReference>();
|
||||||
var candidates = new PriorityQueue<NodeReference, float>();
|
var candidates = new PriorityQueue<NodeReference, float>();
|
||||||
@@ -233,14 +232,13 @@ public sealed class VectorSearchIndex
|
|||||||
{
|
{
|
||||||
float d_c = 0;
|
float d_c = 0;
|
||||||
candidates.TryPeek(out var c, out d_c);
|
candidates.TryPeek(out var c, out d_c);
|
||||||
result.TryPeek(out var f, out var d_f);
|
result.TryPeek(out var f, out float d_f);
|
||||||
|
|
||||||
if (d_c > -d_f) break;
|
if (d_c > -d_f) break;
|
||||||
|
|
||||||
candidates.Dequeue();
|
candidates.Dequeue();
|
||||||
|
|
||||||
foreach (var e in GetNeighbors(c, level, transaction))
|
foreach (var e in GetNeighbors(c, level, transaction))
|
||||||
{
|
|
||||||
if (!visited.Contains(e))
|
if (!visited.Contains(e))
|
||||||
{
|
{
|
||||||
visited.Add(e);
|
visited.Add(e);
|
||||||
@@ -254,7 +252,6 @@ public sealed class VectorSearchIndex
|
|||||||
if (result.Count > ef) result.Dequeue();
|
if (result.Count > ef) result.Dequeue();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert result to list (ordered by distance)
|
// Convert result to list (ordered by distance)
|
||||||
@@ -268,52 +265,53 @@ public sealed class VectorSearchIndex
|
|||||||
{
|
{
|
||||||
// For now, assume a fixed location or track it in page 0 of index
|
// For now, assume a fixed location or track it in page 0 of index
|
||||||
// TODO: Real implementation
|
// TODO: Real implementation
|
||||||
return new NodeReference { PageId = _rootPageId, NodeIndex = 0, MaxLevel = 0 };
|
return new NodeReference { PageId = RootPageId, NodeIndex = 0, MaxLevel = 0 };
|
||||||
}
|
}
|
||||||
|
|
||||||
private float[] LoadVector(NodeReference node, ITransaction? transaction)
|
private float[] LoadVector(NodeReference node, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||||
float[] vector = new float[_options.Dimensions];
|
var vector = new float[_options.Dimensions];
|
||||||
VectorPage.ReadNodeData(buffer, node.NodeIndex, out _, out _, vector);
|
VectorPage.ReadNodeData(buffer, node.NodeIndex, out _, out _, vector);
|
||||||
return vector;
|
return vector;
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Searches the index for the nearest vectors to the query.
|
/// Searches the index for the nearest vectors to the query.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="query">The query vector.</param>
|
/// <param name="query">The query vector.</param>
|
||||||
/// <param name="k">The number of nearest results to return.</param>
|
/// <param name="k">The number of nearest results to return.</param>
|
||||||
/// <param name="efSearch">The search breadth parameter.</param>
|
/// <param name="efSearch">The search breadth parameter.</param>
|
||||||
/// <param name="transaction">Optional transaction context.</param>
|
/// <param name="transaction">Optional transaction context.</param>
|
||||||
/// <returns>The nearest vector search results.</returns>
|
/// <returns>The nearest vector search results.</returns>
|
||||||
public IEnumerable<VectorSearchResult> Search(float[] query, int k, int efSearch = 100, ITransaction? transaction = null)
|
public IEnumerable<VectorSearchResult> Search(float[] query, int k, int efSearch = 100,
|
||||||
|
ITransaction? transaction = null)
|
||||||
{
|
{
|
||||||
if (_rootPageId == 0) yield break;
|
if (RootPageId == 0) yield break;
|
||||||
|
|
||||||
var entryPoint = GetEntryPoint();
|
var entryPoint = GetEntryPoint();
|
||||||
var currentPoint = entryPoint;
|
var currentPoint = entryPoint;
|
||||||
|
|
||||||
// 1. Greedy search through higher layers to find entry point for level 0
|
// 1. Greedy search through higher layers to find entry point for level 0
|
||||||
for (int l = entryPoint.MaxLevel; l > 0; l--)
|
for (int l = entryPoint.MaxLevel; l > 0; l--) currentPoint = GreedySearch(currentPoint, query, l, transaction);
|
||||||
{
|
|
||||||
currentPoint = GreedySearch(currentPoint, query, l, transaction);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Comprehensive search on level 0
|
// 2. Comprehensive search on level 0
|
||||||
var nearest = SearchLayer(currentPoint, query, Math.Max(efSearch, k), 0, transaction);
|
var nearest = SearchLayer(currentPoint, query, Math.Max(efSearch, k), 0, transaction);
|
||||||
|
|
||||||
// 3. Return top-k results
|
// 3. Return top-k results
|
||||||
int count = 0;
|
var count = 0;
|
||||||
foreach (var node in nearest)
|
foreach (var node in nearest)
|
||||||
{
|
{
|
||||||
if (count++ >= k) break;
|
if (count++ >= k) break;
|
||||||
|
|
||||||
float dist = VectorMath.Distance(query, LoadVector(node, transaction), _options.Metric);
|
float dist = VectorMath.Distance(query, LoadVector(node, transaction), _options.Metric);
|
||||||
var loc = LoadDocumentLocation(node, transaction);
|
var loc = LoadDocumentLocation(node, transaction);
|
||||||
yield return new VectorSearchResult(loc, dist);
|
yield return new VectorSearchResult(loc, dist);
|
||||||
@@ -322,34 +320,41 @@ public sealed class VectorSearchIndex
|
|||||||
|
|
||||||
private DocumentLocation LoadDocumentLocation(NodeReference node, ITransaction? transaction)
|
private DocumentLocation LoadDocumentLocation(NodeReference node, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||||
VectorPage.ReadNodeData(buffer, node.NodeIndex, out var loc, out _, new float[0]); // Vector not needed here
|
VectorPage.ReadNodeData(buffer, node.NodeIndex, out var loc, out _, new float[0]); // Vector not needed here
|
||||||
return loc;
|
return loc;
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private IEnumerable<NodeReference> GetNeighbors(NodeReference node, int level, ITransaction? transaction)
|
private IEnumerable<NodeReference> GetNeighbors(NodeReference node, int level, ITransaction? transaction)
|
||||||
{
|
{
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
var results = new List<NodeReference>();
|
var results = new List<NodeReference>();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
_storage.ReadPage(node.PageId, transaction?.TransactionId, buffer);
|
||||||
var links = VectorPage.GetLinksSpan(buffer, node.NodeIndex, level, _options.Dimensions, _options.M);
|
var links = VectorPage.GetLinksSpan(buffer, node.NodeIndex, level, _options.Dimensions, _options.M);
|
||||||
|
|
||||||
for (int i = 0; i < links.Length; i += 6)
|
for (var i = 0; i < links.Length; i += 6)
|
||||||
{
|
{
|
||||||
var loc = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
var loc = DocumentLocation.ReadFrom(links.Slice(i, 6));
|
||||||
if (loc.PageId == 0) break; // End of links
|
if (loc.PageId == 0) break; // End of links
|
||||||
|
|
||||||
results.Add(new NodeReference { PageId = loc.PageId, NodeIndex = loc.SlotIndex });
|
results.Add(new NodeReference { PageId = loc.PageId, NodeIndex = loc.SlotIndex });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -357,29 +362,43 @@ public sealed class VectorSearchIndex
|
|||||||
{
|
{
|
||||||
// Probability p = 1/M for each level
|
// Probability p = 1/M for each level
|
||||||
double p = 1.0 / _options.M;
|
double p = 1.0 / _options.M;
|
||||||
int level = 0;
|
var level = 0;
|
||||||
while (_random.NextDouble() < p && level < 15)
|
while (_random.NextDouble() < p && level < 15) level++;
|
||||||
{
|
|
||||||
level++;
|
|
||||||
}
|
|
||||||
return level;
|
return level;
|
||||||
}
|
}
|
||||||
|
|
||||||
private uint CreateNewPage(ITransaction? transaction)
|
private uint CreateNewPage(ITransaction? transaction)
|
||||||
{
|
{
|
||||||
uint pageId = _storage.AllocatePage();
|
uint pageId = _storage.AllocatePage();
|
||||||
var buffer = RentPageBuffer();
|
byte[] buffer = RentPageBuffer();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
VectorPage.Initialize(buffer, pageId, _options.Dimensions, _options.M);
|
VectorPage.Initialize(buffer, pageId, _options.Dimensions, _options.M);
|
||||||
_storage.WritePageImmediate(pageId, buffer);
|
_storage.WritePageImmediate(pageId, buffer);
|
||||||
return pageId;
|
return pageId;
|
||||||
}
|
}
|
||||||
finally { ReturnPageBuffer(buffer); }
|
finally
|
||||||
|
{
|
||||||
|
ReturnPageBuffer(buffer);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private byte[] RentPageBuffer() => System.Buffers.ArrayPool<byte>.Shared.Rent(_storage.PageSize);
|
private byte[] RentPageBuffer()
|
||||||
private void ReturnPageBuffer(byte[] buffer) => System.Buffers.ArrayPool<byte>.Shared.Return(buffer);
|
{
|
||||||
|
return ArrayPool<byte>.Shared.Rent(_storage.PageSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void ReturnPageBuffer(byte[] buffer)
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
private struct NodeReference
|
||||||
|
{
|
||||||
|
public uint PageId;
|
||||||
|
public int NodeIndex;
|
||||||
|
public int MaxLevel;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public record struct VectorSearchResult(DocumentLocation Location, float Distance);
|
public record struct VectorSearchResult(DocumentLocation Location, float Distance);
|
||||||
@@ -1,42 +1,42 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
using System.Linq.Expressions;
|
||||||
using System.Linq.Expressions;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||||
|
|
||||||
public class EntityTypeBuilder<T> where T : class
|
public class EntityTypeBuilder<T> where T : class
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured collection name for the entity type.
|
/// Gets the configured collection name for the entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string? CollectionName { get; private set; }
|
public string? CollectionName { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured indexes for the entity type.
|
/// Gets the configured indexes for the entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<IndexBuilder<T>> Indexes { get; } = new();
|
public List<IndexBuilder<T>> Indexes { get; } = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the primary key selector expression.
|
/// Gets the primary key selector expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public LambdaExpression? PrimaryKeySelector { get; private set; }
|
public LambdaExpression? PrimaryKeySelector { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether the primary key value is generated on add.
|
/// Gets a value indicating whether the primary key value is generated on add.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool ValueGeneratedOnAdd { get; private set; }
|
public bool ValueGeneratedOnAdd { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured primary key property name.
|
/// Gets the configured primary key property name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string? PrimaryKeyName { get; private set; }
|
public string? PrimaryKeyName { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured property converter types keyed by property name.
|
/// Gets the configured property converter types keyed by property name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Dictionary<string, Type> PropertyConverters { get; } = new();
|
public Dictionary<string, Type> PropertyConverters { get; } = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sets the collection name for the entity type.
|
/// Sets the collection name for the entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The collection name.</param>
|
/// <param name="name">The collection name.</param>
|
||||||
/// <returns>The current entity type builder.</returns>
|
/// <returns>The current entity type builder.</returns>
|
||||||
@@ -47,21 +47,22 @@ public class EntityTypeBuilder<T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds an index for the specified key selector.
|
/// Adds an index for the specified key selector.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">The key type.</typeparam>
|
/// <typeparam name="TKey">The key type.</typeparam>
|
||||||
/// <param name="keySelector">The key selector expression.</param>
|
/// <param name="keySelector">The key selector expression.</param>
|
||||||
/// <param name="name">The optional index name.</param>
|
/// <param name="name">The optional index name.</param>
|
||||||
/// <param name="unique">A value indicating whether the index is unique.</param>
|
/// <param name="unique">A value indicating whether the index is unique.</param>
|
||||||
/// <returns>The current entity type builder.</returns>
|
/// <returns>The current entity type builder.</returns>
|
||||||
public EntityTypeBuilder<T> HasIndex<TKey>(Expression<Func<T, TKey>> keySelector, string? name = null, bool unique = false)
|
public EntityTypeBuilder<T> HasIndex<TKey>(Expression<Func<T, TKey>> keySelector, string? name = null,
|
||||||
|
bool unique = false)
|
||||||
{
|
{
|
||||||
Indexes.Add(new IndexBuilder<T>(keySelector, name, unique));
|
Indexes.Add(new IndexBuilder<T>(keySelector, name, unique));
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a vector index for the specified key selector.
|
/// Adds a vector index for the specified key selector.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">The key type.</typeparam>
|
/// <typeparam name="TKey">The key type.</typeparam>
|
||||||
/// <param name="keySelector">The key selector expression.</param>
|
/// <param name="keySelector">The key selector expression.</param>
|
||||||
@@ -69,14 +70,15 @@ public class EntityTypeBuilder<T> where T : class
|
|||||||
/// <param name="metric">The vector similarity metric.</param>
|
/// <param name="metric">The vector similarity metric.</param>
|
||||||
/// <param name="name">The optional index name.</param>
|
/// <param name="name">The optional index name.</param>
|
||||||
/// <returns>The current entity type builder.</returns>
|
/// <returns>The current entity type builder.</returns>
|
||||||
public EntityTypeBuilder<T> HasVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector, int dimensions, VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
public EntityTypeBuilder<T> HasVectorIndex<TKey>(Expression<Func<T, TKey>> keySelector, int dimensions,
|
||||||
|
VectorMetric metric = VectorMetric.Cosine, string? name = null)
|
||||||
{
|
{
|
||||||
Indexes.Add(new IndexBuilder<T>(keySelector, name, false, IndexType.Vector, dimensions, metric));
|
Indexes.Add(new IndexBuilder<T>(keySelector, name, false, IndexType.Vector, dimensions, metric));
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a spatial index for the specified key selector.
|
/// Adds a spatial index for the specified key selector.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">The key type.</typeparam>
|
/// <typeparam name="TKey">The key type.</typeparam>
|
||||||
/// <param name="keySelector">The key selector expression.</param>
|
/// <param name="keySelector">The key selector expression.</param>
|
||||||
@@ -89,7 +91,7 @@ public class EntityTypeBuilder<T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Sets the primary key selector for the entity type.
|
/// Sets the primary key selector for the entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TKey">The key type.</typeparam>
|
/// <typeparam name="TKey">The key type.</typeparam>
|
||||||
/// <param name="keySelector">The primary key selector expression.</param>
|
/// <param name="keySelector">The primary key selector expression.</param>
|
||||||
@@ -102,38 +104,35 @@ public class EntityTypeBuilder<T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Configures a converter for the primary key property.
|
/// Configures a converter for the primary key property.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TConverter">The converter type.</typeparam>
|
/// <typeparam name="TConverter">The converter type.</typeparam>
|
||||||
/// <returns>The current entity type builder.</returns>
|
/// <returns>The current entity type builder.</returns>
|
||||||
public EntityTypeBuilder<T> HasConversion<TConverter>()
|
public EntityTypeBuilder<T> HasConversion<TConverter>()
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrEmpty(PrimaryKeyName))
|
if (!string.IsNullOrEmpty(PrimaryKeyName)) PropertyConverters[PrimaryKeyName] = typeof(TConverter);
|
||||||
{
|
|
||||||
PropertyConverters[PrimaryKeyName] = typeof(TConverter);
|
|
||||||
}
|
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Configures a specific property on the entity type.
|
/// Configures a specific property on the entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TProperty">The property type.</typeparam>
|
/// <typeparam name="TProperty">The property type.</typeparam>
|
||||||
/// <param name="propertyExpression">The property expression.</param>
|
/// <param name="propertyExpression">The property expression.</param>
|
||||||
/// <returns>A builder for the selected property.</returns>
|
/// <returns>A builder for the selected property.</returns>
|
||||||
public PropertyBuilder Property<TProperty>(Expression<Func<T, TProperty>> propertyExpression)
|
public PropertyBuilder Property<TProperty>(Expression<Func<T, TProperty>> propertyExpression)
|
||||||
{
|
{
|
||||||
var propertyName = ExpressionAnalyzer.ExtractPropertyPaths(propertyExpression).FirstOrDefault();
|
string? propertyName = ExpressionAnalyzer.ExtractPropertyPaths(propertyExpression).FirstOrDefault();
|
||||||
return new PropertyBuilder(this, propertyName);
|
return new PropertyBuilder(this, propertyName);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class PropertyBuilder
|
public class PropertyBuilder
|
||||||
{
|
{
|
||||||
private readonly EntityTypeBuilder<T> _parent;
|
private readonly EntityTypeBuilder<T> _parent;
|
||||||
private readonly string? _propertyName;
|
private readonly string? _propertyName;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="PropertyBuilder"/> class.
|
/// Initializes a new instance of the <see cref="PropertyBuilder" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="parent">The parent entity type builder.</param>
|
/// <param name="parent">The parent entity type builder.</param>
|
||||||
/// <param name="propertyName">The property name.</param>
|
/// <param name="propertyName">The property name.</param>
|
||||||
@@ -144,68 +143,32 @@ public class EntityTypeBuilder<T> where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks the configured property as value generated on add.
|
/// Marks the configured property as value generated on add.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The current property builder.</returns>
|
/// <returns>The current property builder.</returns>
|
||||||
public PropertyBuilder ValueGeneratedOnAdd()
|
public PropertyBuilder ValueGeneratedOnAdd()
|
||||||
{
|
{
|
||||||
if (_propertyName == _parent.PrimaryKeyName)
|
if (_propertyName == _parent.PrimaryKeyName) _parent.ValueGeneratedOnAdd = true;
|
||||||
{
|
|
||||||
_parent.ValueGeneratedOnAdd = true;
|
|
||||||
}
|
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Configures a converter for the configured property.
|
/// Configures a converter for the configured property.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TConverter">The converter type.</typeparam>
|
/// <typeparam name="TConverter">The converter type.</typeparam>
|
||||||
/// <returns>The current property builder.</returns>
|
/// <returns>The current property builder.</returns>
|
||||||
public PropertyBuilder HasConversion<TConverter>()
|
public PropertyBuilder HasConversion<TConverter>()
|
||||||
{
|
{
|
||||||
if (!string.IsNullOrEmpty(_propertyName))
|
if (!string.IsNullOrEmpty(_propertyName)) _parent.PropertyConverters[_propertyName] = typeof(TConverter);
|
||||||
{
|
return this;
|
||||||
_parent.PropertyConverters[_propertyName] = typeof(TConverter);
|
}
|
||||||
}
|
}
|
||||||
return this;
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public class IndexBuilder<T>
|
public class IndexBuilder<T>
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the index key selector expression.
|
/// Initializes a new instance of the <see cref="IndexBuilder{T}" /> class.
|
||||||
/// </summary>
|
|
||||||
public LambdaExpression KeySelector { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the configured index name.
|
|
||||||
/// </summary>
|
|
||||||
public string? Name { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets a value indicating whether the index is unique.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsUnique { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the index type.
|
|
||||||
/// </summary>
|
|
||||||
public IndexType Type { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the vector dimensions.
|
|
||||||
/// </summary>
|
|
||||||
public int Dimensions { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the vector metric.
|
|
||||||
/// </summary>
|
|
||||||
public VectorMetric Metric { get; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Initializes a new instance of the <see cref="IndexBuilder{T}"/> class.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keySelector">The index key selector expression.</param>
|
/// <param name="keySelector">The index key selector expression.</param>
|
||||||
/// <param name="name">The optional index name.</param>
|
/// <param name="name">The optional index name.</param>
|
||||||
@@ -213,13 +176,44 @@ public class IndexBuilder<T>
|
|||||||
/// <param name="type">The index type.</param>
|
/// <param name="type">The index type.</param>
|
||||||
/// <param name="dimensions">The vector dimensions.</param>
|
/// <param name="dimensions">The vector dimensions.</param>
|
||||||
/// <param name="metric">The vector metric.</param>
|
/// <param name="metric">The vector metric.</param>
|
||||||
public IndexBuilder(LambdaExpression keySelector, string? name, bool unique, IndexType type = IndexType.BTree, int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
public IndexBuilder(LambdaExpression keySelector, string? name, bool unique, IndexType type = IndexType.BTree,
|
||||||
|
int dimensions = 0, VectorMetric metric = VectorMetric.Cosine)
|
||||||
{
|
{
|
||||||
KeySelector = keySelector;
|
KeySelector = keySelector;
|
||||||
Name = name;
|
Name = name;
|
||||||
IsUnique = unique;
|
IsUnique = unique;
|
||||||
Type = type;
|
Type = type;
|
||||||
Dimensions = dimensions;
|
Dimensions = dimensions;
|
||||||
Metric = metric;
|
Metric = metric;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the index key selector expression.
|
||||||
|
/// </summary>
|
||||||
|
public LambdaExpression KeySelector { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the configured index name.
|
||||||
|
/// </summary>
|
||||||
|
public string? Name { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a value indicating whether the index is unique.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsUnique { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the index type.
|
||||||
|
/// </summary>
|
||||||
|
public IndexType Type { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the vector dimensions.
|
||||||
|
/// </summary>
|
||||||
|
public int Dimensions { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the vector metric.
|
||||||
|
/// </summary>
|
||||||
|
public VectorMetric Metric { get; }
|
||||||
|
}
|
||||||
@@ -1,30 +1,31 @@
|
|||||||
using System.Linq.Expressions;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||||
|
|
||||||
public class ModelBuilder
|
public class ModelBuilder
|
||||||
{
|
{
|
||||||
private readonly Dictionary<Type, object> _entityBuilders = new();
|
private readonly Dictionary<Type, object> _entityBuilders = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or creates the entity builder for the specified entity type.
|
/// Gets or creates the entity builder for the specified entity type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="T">The entity type.</typeparam>
|
/// <typeparam name="T">The entity type.</typeparam>
|
||||||
/// <returns>The entity builder for <typeparamref name="T"/>.</returns>
|
/// <returns>The entity builder for <typeparamref name="T" />.</returns>
|
||||||
public EntityTypeBuilder<T> Entity<T>() where T : class
|
public EntityTypeBuilder<T> Entity<T>() where T : class
|
||||||
{
|
{
|
||||||
if (!_entityBuilders.TryGetValue(typeof(T), out var builder))
|
if (!_entityBuilders.TryGetValue(typeof(T), out object? builder))
|
||||||
{
|
{
|
||||||
builder = new EntityTypeBuilder<T>();
|
builder = new EntityTypeBuilder<T>();
|
||||||
_entityBuilders[typeof(T)] = builder;
|
_entityBuilders[typeof(T)] = builder;
|
||||||
}
|
}
|
||||||
|
|
||||||
return (EntityTypeBuilder<T>)builder;
|
return (EntityTypeBuilder<T>)builder;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets all registered entity builders.
|
/// Gets all registered entity builders.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>A read-only dictionary of entity builders keyed by entity type.</returns>
|
/// <returns>A read-only dictionary of entity builders keyed by entity type.</returns>
|
||||||
public IReadOnlyDictionary<Type, object> GetEntityBuilders() => _entityBuilders;
|
public IReadOnlyDictionary<Type, object> GetEntityBuilders()
|
||||||
}
|
{
|
||||||
|
return _entityBuilders;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,20 +1,20 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
namespace ZB.MOM.WW.CBDD.Core.Metadata;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Defines a bidirectional conversion between a model type (e.g. ValueObject)
|
/// Defines a bidirectional conversion between a model type (e.g. ValueObject)
|
||||||
/// and a provider type supported by the storage engine (e.g. string, int, Guid, ObjectId).
|
/// and a provider type supported by the storage engine (e.g. string, int, Guid, ObjectId).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public abstract class ValueConverter<TModel, TProvider>
|
public abstract class ValueConverter<TModel, TProvider>
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts the model value to the provider value.
|
/// Converts the model value to the provider value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="model">The model value to convert.</param>
|
/// <param name="model">The model value to convert.</param>
|
||||||
public abstract TProvider ConvertToProvider(TModel model);
|
public abstract TProvider ConvertToProvider(TModel model);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Converts the provider value back to the model value.
|
/// Converts the provider value back to the model value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="provider">The provider value to convert.</param>
|
/// <param name="provider">The provider value to convert.</param>
|
||||||
public abstract TModel ConvertFromProvider(TProvider provider);
|
public abstract TModel ConvertFromProvider(TProvider provider);
|
||||||
}
|
}
|
||||||
@@ -4,18 +4,20 @@ namespace ZB.MOM.WW.CBDD.Core.Query;
|
|||||||
|
|
||||||
internal class BTreeExpressionVisitor : ExpressionVisitor
|
internal class BTreeExpressionVisitor : ExpressionVisitor
|
||||||
{
|
{
|
||||||
private readonly QueryModel _model = new();
|
private readonly QueryModel _model = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the query model built while visiting an expression tree.
|
/// Gets the query model built while visiting an expression tree.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public QueryModel GetModel() => _model;
|
public QueryModel GetModel()
|
||||||
|
{
|
||||||
/// <inheritdoc />
|
return _model;
|
||||||
protected override Expression VisitMethodCall(MethodCallExpression node)
|
}
|
||||||
{
|
|
||||||
if (node.Method.DeclaringType == typeof(Queryable))
|
/// <inheritdoc />
|
||||||
{
|
protected override Expression VisitMethodCall(MethodCallExpression node)
|
||||||
|
{
|
||||||
|
if (node.Method.DeclaringType == typeof(Queryable))
|
||||||
switch (node.Method.Name)
|
switch (node.Method.Name)
|
||||||
{
|
{
|
||||||
case "Where":
|
case "Where":
|
||||||
@@ -35,8 +37,7 @@ internal class BTreeExpressionVisitor : ExpressionVisitor
|
|||||||
VisitSkip(node);
|
VisitSkip(node);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return base.VisitMethodCall(node);
|
return base.VisitMethodCall(node);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -94,4 +95,4 @@ internal class BTreeExpressionVisitor : ExpressionVisitor
|
|||||||
if (countExpression.Value != null)
|
if (countExpression.Value != null)
|
||||||
_model.Skip = (int)countExpression.Value;
|
_model.Skip = (int)countExpression.Value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,17 +1,17 @@
|
|||||||
using System.Linq;
|
using System.Linq.Expressions;
|
||||||
using System.Linq.Expressions;
|
using System.Reflection;
|
||||||
using System.Reflection;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using static ZB.MOM.WW.CBDD.Core.Query.IndexOptimizer;
|
using static ZB.MOM.WW.CBDD.Core.Query.IndexOptimizer;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||||
|
|
||||||
public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
||||||
{
|
{
|
||||||
private readonly DocumentCollection<TId, T> _collection;
|
private readonly DocumentCollection<TId, T> _collection;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="BTreeQueryProvider{TId, T}"/> class.
|
/// Initializes a new instance of the <see cref="BTreeQueryProvider{TId, T}" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="collection">The backing document collection.</param>
|
/// <param name="collection">The backing document collection.</param>
|
||||||
public BTreeQueryProvider(DocumentCollection<TId, T> collection)
|
public BTreeQueryProvider(DocumentCollection<TId, T> collection)
|
||||||
@@ -20,38 +20,37 @@ public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a query from the specified expression.
|
/// Creates a query from the specified expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="expression">The query expression.</param>
|
/// <param name="expression">The query expression.</param>
|
||||||
/// <returns>An <see cref="IQueryable"/> representing the query.</returns>
|
/// <returns>An <see cref="IQueryable" /> representing the query.</returns>
|
||||||
public IQueryable CreateQuery(Expression expression)
|
public IQueryable CreateQuery(Expression expression)
|
||||||
{
|
{
|
||||||
var elementType = expression.Type.GetGenericArguments()[0];
|
var elementType = expression.Type.GetGenericArguments()[0];
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
return (IQueryable)Activator.CreateInstance(
|
return (IQueryable)Activator.CreateInstance(
|
||||||
typeof(BTreeQueryable<>).MakeGenericType(elementType),
|
typeof(BTreeQueryable<>).MakeGenericType(elementType), this, expression)!;
|
||||||
new object[] { this, expression })!;
|
}
|
||||||
}
|
catch (TargetInvocationException ex)
|
||||||
catch (TargetInvocationException ex)
|
{
|
||||||
{
|
|
||||||
throw ex.InnerException ?? ex;
|
throw ex.InnerException ?? ex;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates a strongly typed query from the specified expression.
|
/// Creates a strongly typed query from the specified expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TElement">The element type of the query.</typeparam>
|
/// <typeparam name="TElement">The element type of the query.</typeparam>
|
||||||
/// <param name="expression">The query expression.</param>
|
/// <param name="expression">The query expression.</param>
|
||||||
/// <returns>An <see cref="IQueryable{T}"/> representing the query.</returns>
|
/// <returns>An <see cref="IQueryable{T}" /> representing the query.</returns>
|
||||||
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
|
public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
|
||||||
{
|
{
|
||||||
return new BTreeQueryable<TElement>(this, expression);
|
return new BTreeQueryable<TElement>(this, expression);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes a query expression.
|
/// Executes a query expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="expression">The query expression.</param>
|
/// <param name="expression">The query expression.</param>
|
||||||
/// <returns>The query result.</returns>
|
/// <returns>The query result.</returns>
|
||||||
@@ -61,7 +60,7 @@ public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes a query expression and returns a strongly typed result.
|
/// Executes a query expression and returns a strongly typed result.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="TResult">The result type.</typeparam>
|
/// <typeparam name="TResult">The result type.</typeparam>
|
||||||
/// <param name="expression">The query expression.</param>
|
/// <param name="expression">The query expression.</param>
|
||||||
@@ -72,88 +71,71 @@ public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
|||||||
// We only care about WHERE clause for optimization.
|
// We only care about WHERE clause for optimization.
|
||||||
// GroupBy, Select, OrderBy, etc. are handled by EnumerableRewriter.
|
// GroupBy, Select, OrderBy, etc. are handled by EnumerableRewriter.
|
||||||
|
|
||||||
var visitor = new BTreeExpressionVisitor();
|
var visitor = new BTreeExpressionVisitor();
|
||||||
visitor.Visit(expression);
|
visitor.Visit(expression);
|
||||||
var model = visitor.GetModel();
|
var model = visitor.GetModel();
|
||||||
|
|
||||||
// 2. Data Fetching Strategy (Optimized or Full Scan)
|
// 2. Data Fetching Strategy (Optimized or Full Scan)
|
||||||
IEnumerable<T> sourceData = null!;
|
IEnumerable<T> sourceData = null!;
|
||||||
|
|
||||||
// A. Try Index Optimization (Only if Where clause exists)
|
// A. Try Index Optimization (Only if Where clause exists)
|
||||||
var indexOpt = IndexOptimizer.TryOptimize<T>(model, _collection.GetIndexes());
|
var indexOpt = TryOptimize<T>(model, _collection.GetIndexes());
|
||||||
if (indexOpt != null)
|
if (indexOpt != null)
|
||||||
{
|
{
|
||||||
if (indexOpt.IsVectorSearch)
|
if (indexOpt.IsVectorSearch)
|
||||||
{
|
|
||||||
sourceData = _collection.VectorSearch(indexOpt.IndexName, indexOpt.VectorQuery!, indexOpt.K);
|
sourceData = _collection.VectorSearch(indexOpt.IndexName, indexOpt.VectorQuery!, indexOpt.K);
|
||||||
}
|
|
||||||
else if (indexOpt.IsSpatialSearch)
|
else if (indexOpt.IsSpatialSearch)
|
||||||
{
|
|
||||||
sourceData = indexOpt.SpatialType == SpatialQueryType.Near
|
sourceData = indexOpt.SpatialType == SpatialQueryType.Near
|
||||||
? _collection.Near(indexOpt.IndexName, indexOpt.SpatialPoint, indexOpt.RadiusKm)
|
? _collection.Near(indexOpt.IndexName, indexOpt.SpatialPoint, indexOpt.RadiusKm)
|
||||||
: _collection.Within(indexOpt.IndexName, indexOpt.SpatialMin, indexOpt.SpatialMax);
|
: _collection.Within(indexOpt.IndexName, indexOpt.SpatialMin, indexOpt.SpatialMax);
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
|
||||||
sourceData = _collection.QueryIndex(indexOpt.IndexName, indexOpt.MinValue, indexOpt.MaxValue);
|
sourceData = _collection.QueryIndex(indexOpt.IndexName, indexOpt.MinValue, indexOpt.MaxValue);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// B. Try Scan Optimization (if no index used)
|
// B. Try Scan Optimization (if no index used)
|
||||||
if (sourceData == null)
|
if (sourceData == null)
|
||||||
{
|
{
|
||||||
Func<ZB.MOM.WW.CBDD.Bson.BsonSpanReader, bool>? bsonPredicate = null;
|
Func<BsonSpanReader, bool>? bsonPredicate = null;
|
||||||
if (model.WhereClause != null)
|
if (model.WhereClause != null) bsonPredicate = BsonExpressionEvaluator.TryCompile<T>(model.WhereClause);
|
||||||
{
|
|
||||||
bsonPredicate = BsonExpressionEvaluator.TryCompile<T>(model.WhereClause);
|
if (bsonPredicate != null) sourceData = _collection.Scan(bsonPredicate);
|
||||||
}
|
|
||||||
|
|
||||||
if (bsonPredicate != null)
|
|
||||||
{
|
|
||||||
sourceData = _collection.Scan(bsonPredicate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// C. Fallback to Full Scan
|
|
||||||
if (sourceData == null)
|
|
||||||
{
|
|
||||||
sourceData = _collection.FindAll();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// C. Fallback to Full Scan
|
||||||
|
if (sourceData == null) sourceData = _collection.FindAll();
|
||||||
|
|
||||||
// 3. Rewrite Expression Tree to use Enumerable
|
// 3. Rewrite Expression Tree to use Enumerable
|
||||||
// Replace the "Root" IQueryable with our sourceData IEnumerable
|
// Replace the "Root" IQueryable with our sourceData IEnumerable
|
||||||
|
|
||||||
// We need to find the root IQueryable in the expression to replace it.
|
// We need to find the root IQueryable in the expression to replace it.
|
||||||
// It's likely the first argument of the first method call, or a constant.
|
// It's likely the first argument of the first method call, or a constant.
|
||||||
|
|
||||||
var rootFinder = new RootFinder();
|
var rootFinder = new RootFinder();
|
||||||
rootFinder.Visit(expression);
|
rootFinder.Visit(expression);
|
||||||
var root = rootFinder.Root;
|
var root = rootFinder.Root;
|
||||||
|
|
||||||
if (root == null) throw new InvalidOperationException("Could not find root Queryable in expression");
|
if (root == null) throw new InvalidOperationException("Could not find root Queryable in expression");
|
||||||
|
|
||||||
var rewriter = new EnumerableRewriter(root, sourceData);
|
var rewriter = new EnumerableRewriter(root, sourceData);
|
||||||
var rewrittenExpression = rewriter.Visit(expression);
|
var rewrittenExpression = rewriter.Visit(expression);
|
||||||
|
|
||||||
// 4. Compile and Execute
|
// 4. Compile and Execute
|
||||||
// The rewritten expression is now a tree of IEnumerable calls returning TResult.
|
// The rewritten expression is now a tree of IEnumerable calls returning TResult.
|
||||||
// We need to turn it into a Func<TResult> and invoke it.
|
// We need to turn it into a Func<TResult> and invoke it.
|
||||||
|
|
||||||
if (rewrittenExpression.Type != typeof(TResult))
|
if (rewrittenExpression.Type != typeof(TResult))
|
||||||
{
|
|
||||||
// If TResult is object (non-generic Execute), we need to cast
|
// If TResult is object (non-generic Execute), we need to cast
|
||||||
rewrittenExpression = Expression.Convert(rewrittenExpression, typeof(TResult));
|
rewrittenExpression = Expression.Convert(rewrittenExpression, typeof(TResult));
|
||||||
}
|
|
||||||
|
var lambda = Expression.Lambda<Func<TResult>>(rewrittenExpression);
|
||||||
var lambda = Expression.Lambda<Func<TResult>>(rewrittenExpression);
|
var compiled = lambda.Compile();
|
||||||
var compiled = lambda.Compile();
|
return compiled();
|
||||||
return compiled();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private class RootFinder : ExpressionVisitor
|
private class RootFinder : ExpressionVisitor
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the root queryable found in the expression tree.
|
/// Gets the root queryable found in the expression tree.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IQueryable? Root { get; private set; }
|
public IQueryable? Root { get; private set; }
|
||||||
|
|
||||||
@@ -161,13 +143,11 @@ public class BTreeQueryProvider<TId, T> : IQueryProvider where T : class
|
|||||||
protected override Expression VisitConstant(ConstantExpression node)
|
protected override Expression VisitConstant(ConstantExpression node)
|
||||||
{
|
{
|
||||||
// If we found a Queryable, that's our root source
|
// If we found a Queryable, that's our root source
|
||||||
if (Root == null && node.Value is IQueryable q)
|
if (Root == null && node.Value is IQueryable q)
|
||||||
{
|
// We typically want the "base" queryable (the BTreeQueryable instance)
|
||||||
// We typically want the "base" queryable (the BTreeQueryable instance)
|
// In a chain like Coll.Where.Select, the root is Coll.
|
||||||
// In a chain like Coll.Where.Select, the root is Coll.
|
Root = q;
|
||||||
Root = q;
|
return base.VisitConstant(node);
|
||||||
}
|
}
|
||||||
return base.VisitConstant(node);
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,13 +1,12 @@
|
|||||||
using System.Collections;
|
using System.Collections;
|
||||||
using System.Linq;
|
using System.Linq.Expressions;
|
||||||
using System.Linq.Expressions;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
|
||||||
|
|
||||||
internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new queryable wrapper for the specified provider and expression.
|
/// Initializes a new queryable wrapper for the specified provider and expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="provider">The query provider.</param>
|
/// <param name="provider">The query provider.</param>
|
||||||
/// <param name="expression">The expression tree.</param>
|
/// <param name="expression">The expression tree.</param>
|
||||||
@@ -18,7 +17,7 @@ internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new queryable wrapper for the specified provider.
|
/// Initializes a new queryable wrapper for the specified provider.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="provider">The query provider.</param>
|
/// <param name="provider">The query provider.</param>
|
||||||
public BTreeQueryable(IQueryProvider provider)
|
public BTreeQueryable(IQueryProvider provider)
|
||||||
@@ -28,17 +27,17 @@ internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the element type returned by this query.
|
/// Gets the element type returned by this query.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Type ElementType => typeof(T);
|
public Type ElementType => typeof(T);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the expression tree associated with this query.
|
/// Gets the expression tree associated with this query.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Expression Expression { get; }
|
public Expression Expression { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the query provider for this query.
|
/// Gets the query provider for this query.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IQueryProvider Provider { get; }
|
public IQueryProvider Provider { get; }
|
||||||
|
|
||||||
@@ -53,4 +52,4 @@ internal class BTreeQueryable<T> : IOrderedQueryable<T>
|
|||||||
{
|
{
|
||||||
return GetEnumerator();
|
return GetEnumerator();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -6,11 +6,11 @@ namespace ZB.MOM.WW.CBDD.Core.Query;
|
|||||||
internal static class BsonExpressionEvaluator
|
internal static class BsonExpressionEvaluator
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Attempts to compile a LINQ predicate expression into a BSON reader predicate.
|
/// Attempts to compile a LINQ predicate expression into a BSON reader predicate.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <typeparam name="T">The entity type of the original expression.</typeparam>
|
/// <typeparam name="T">The entity type of the original expression.</typeparam>
|
||||||
/// <param name="expression">The lambda expression to compile.</param>
|
/// <param name="expression">The lambda expression to compile.</param>
|
||||||
/// <returns>A compiled predicate when supported; otherwise, <see langword="null"/>.</returns>
|
/// <returns>A compiled predicate when supported; otherwise, <see langword="null" />.</returns>
|
||||||
public static Func<BsonSpanReader, bool>? TryCompile<T>(LambdaExpression expression)
|
public static Func<BsonSpanReader, bool>? TryCompile<T>(LambdaExpression expression)
|
||||||
{
|
{
|
||||||
// Simple optimization for: x => x.Prop op Constant
|
// Simple optimization for: x => x.Prop op Constant
|
||||||
@@ -29,12 +29,11 @@ internal static class BsonExpressionEvaluator
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (left is MemberExpression member && right is ConstantExpression constant)
|
if (left is MemberExpression member && right is ConstantExpression constant)
|
||||||
{
|
|
||||||
// Check if member is property of parameter
|
// Check if member is property of parameter
|
||||||
if (member.Expression == expression.Parameters[0])
|
if (member.Expression == expression.Parameters[0])
|
||||||
{
|
{
|
||||||
var propertyName = member.Member.Name.ToLowerInvariant();
|
string propertyName = member.Member.Name.ToLowerInvariant();
|
||||||
var value = constant.Value;
|
object? value = constant.Value;
|
||||||
|
|
||||||
// Handle Id mapping?
|
// Handle Id mapping?
|
||||||
// If property is "id", Bson field is "_id"
|
// If property is "id", Bson field is "_id"
|
||||||
@@ -42,22 +41,25 @@ internal static class BsonExpressionEvaluator
|
|||||||
|
|
||||||
return CreatePredicate(propertyName, value, nodeType);
|
return CreatePredicate(propertyName, value, nodeType);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ExpressionType Flip(ExpressionType type) => type switch
|
private static ExpressionType Flip(ExpressionType type)
|
||||||
{
|
{
|
||||||
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
return type switch
|
||||||
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
{
|
||||||
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
||||||
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
||||||
_ => type
|
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
||||||
};
|
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
||||||
|
_ => type
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private static Func<BsonSpanReader, bool>? CreatePredicate(string propertyName, object? targetValue, ExpressionType op)
|
private static Func<BsonSpanReader, bool>? CreatePredicate(string propertyName, object? targetValue,
|
||||||
|
ExpressionType op)
|
||||||
{
|
{
|
||||||
// We need to return a delegate that searches for propertyName in BsonSpanReader and compares
|
// We need to return a delegate that searches for propertyName in BsonSpanReader and compares
|
||||||
|
|
||||||
@@ -71,13 +73,11 @@ internal static class BsonExpressionEvaluator
|
|||||||
var type = reader.ReadBsonType();
|
var type = reader.ReadBsonType();
|
||||||
if (type == 0) break;
|
if (type == 0) break;
|
||||||
|
|
||||||
var name = reader.ReadElementHeader();
|
string name = reader.ReadElementHeader();
|
||||||
|
|
||||||
if (name == propertyName)
|
if (name == propertyName)
|
||||||
{
|
|
||||||
// Found -> read value and compare
|
// Found -> read value and compare
|
||||||
return Compare(ref reader, type, targetValue, op);
|
return Compare(ref reader, type, targetValue, op);
|
||||||
}
|
|
||||||
|
|
||||||
reader.SkipValue(type);
|
reader.SkipValue(type);
|
||||||
}
|
}
|
||||||
@@ -86,6 +86,7 @@ internal static class BsonExpressionEvaluator
|
|||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false; // Not found
|
return false; // Not found
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -97,9 +98,8 @@ internal static class BsonExpressionEvaluator
|
|||||||
|
|
||||||
if (type == BsonType.Int32)
|
if (type == BsonType.Int32)
|
||||||
{
|
{
|
||||||
var val = reader.ReadInt32();
|
int val = reader.ReadInt32();
|
||||||
if (target is int targetInt)
|
if (target is int targetInt)
|
||||||
{
|
|
||||||
return op switch
|
return op switch
|
||||||
{
|
{
|
||||||
ExpressionType.Equal => val == targetInt,
|
ExpressionType.Equal => val == targetInt,
|
||||||
@@ -110,14 +110,13 @@ internal static class BsonExpressionEvaluator
|
|||||||
ExpressionType.LessThanOrEqual => val <= targetInt,
|
ExpressionType.LessThanOrEqual => val <= targetInt,
|
||||||
_ => false
|
_ => false
|
||||||
};
|
};
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (type == BsonType.String)
|
else if (type == BsonType.String)
|
||||||
{
|
{
|
||||||
var val = reader.ReadString();
|
string val = reader.ReadString();
|
||||||
if (target is string targetStr)
|
if (target is string targetStr)
|
||||||
{
|
{
|
||||||
var cmp = string.Compare(val, targetStr, StringComparison.Ordinal);
|
int cmp = string.Compare(val, targetStr, StringComparison.Ordinal);
|
||||||
return op switch
|
return op switch
|
||||||
{
|
{
|
||||||
ExpressionType.Equal => cmp == 0,
|
ExpressionType.Equal => cmp == 0,
|
||||||
@@ -140,4 +139,4 @@ internal static class BsonExpressionEvaluator
|
|||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,3 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Linq.Expressions;
|
using System.Linq.Expressions;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
|
|
||||||
@@ -8,50 +5,48 @@ namespace ZB.MOM.WW.CBDD.Core.Query;
|
|||||||
|
|
||||||
internal class EnumerableRewriter : ExpressionVisitor
|
internal class EnumerableRewriter : ExpressionVisitor
|
||||||
{
|
{
|
||||||
private readonly IQueryable _source;
|
private readonly IQueryable _source;
|
||||||
private readonly object _target;
|
private readonly object _target;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="EnumerableRewriter"/> class.
|
/// Initializes a new instance of the <see cref="EnumerableRewriter" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The original queryable source to replace.</param>
|
/// <param name="source">The original queryable source to replace.</param>
|
||||||
/// <param name="target">The target enumerable-backed object.</param>
|
/// <param name="target">The target enumerable-backed object.</param>
|
||||||
public EnumerableRewriter(IQueryable source, object target)
|
public EnumerableRewriter(IQueryable source, object target)
|
||||||
{
|
{
|
||||||
_source = source;
|
_source = source;
|
||||||
_target = target;
|
_target = target;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
protected override Expression VisitConstant(ConstantExpression node)
|
protected override Expression VisitConstant(ConstantExpression node)
|
||||||
{
|
{
|
||||||
// Replace the IQueryable source with the materialized IEnumerable
|
// Replace the IQueryable source with the materialized IEnumerable
|
||||||
if (node.Value == _source)
|
if (node.Value == _source) return Expression.Constant(_target);
|
||||||
{
|
|
||||||
return Expression.Constant(_target);
|
|
||||||
}
|
|
||||||
return base.VisitConstant(node);
|
return base.VisitConstant(node);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
protected override Expression VisitMethodCall(MethodCallExpression node)
|
protected override Expression VisitMethodCall(MethodCallExpression node)
|
||||||
{
|
{
|
||||||
if (node.Method.DeclaringType == typeof(Queryable))
|
if (node.Method.DeclaringType == typeof(Queryable))
|
||||||
{
|
{
|
||||||
var methodName = node.Method.Name;
|
string methodName = node.Method.Name;
|
||||||
var typeArgs = node.Method.GetGenericArguments();
|
var typeArgs = node.Method.GetGenericArguments();
|
||||||
var args = new Expression[node.Arguments.Count];
|
var args = new Expression[node.Arguments.Count];
|
||||||
|
|
||||||
for (int i = 0; i < node.Arguments.Count; i++)
|
for (var i = 0; i < node.Arguments.Count; i++)
|
||||||
{
|
{
|
||||||
var arg = Visit(node.Arguments[i]);
|
var arg = Visit(node.Arguments[i]);
|
||||||
|
|
||||||
// Strip Quote from lambda arguments
|
// Strip Quote from lambda arguments
|
||||||
if (arg is UnaryExpression quote && quote.NodeType == ExpressionType.Quote)
|
if (arg is UnaryExpression quote && quote.NodeType == ExpressionType.Quote)
|
||||||
{
|
{
|
||||||
var lambda = (LambdaExpression)quote.Operand;
|
var lambda = (LambdaExpression)quote.Operand;
|
||||||
arg = Expression.Constant(lambda.Compile());
|
arg = Expression.Constant(lambda.Compile());
|
||||||
}
|
}
|
||||||
|
|
||||||
args[i] = arg;
|
args[i] = arg;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,4 +78,4 @@ internal class EnumerableRewriter : ExpressionVisitor
|
|||||||
|
|
||||||
return base.VisitMethodCall(node);
|
return base.VisitMethodCall(node);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,96 +3,30 @@ using ZB.MOM.WW.CBDD.Core.Indexing;
|
|||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||||
|
|
||||||
internal static class IndexOptimizer
|
internal static class IndexOptimizer
|
||||||
{
|
{
|
||||||
/// <summary>
|
public enum SpatialQueryType
|
||||||
/// Represents the selected index and bounds for an optimized query.
|
{
|
||||||
/// </summary>
|
Near,
|
||||||
public class OptimizationResult
|
Within
|
||||||
{
|
}
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the selected index name.
|
|
||||||
/// </summary>
|
|
||||||
public string IndexName { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the minimum bound value.
|
|
||||||
/// </summary>
|
|
||||||
public object? MinValue { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the maximum bound value.
|
|
||||||
/// </summary>
|
|
||||||
public object? MaxValue { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the query uses a range.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsRange { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the query uses vector search.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsVectorSearch { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the vector query values.
|
|
||||||
/// </summary>
|
|
||||||
public float[]? VectorQuery { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the number of nearest neighbors for vector search.
|
|
||||||
/// </summary>
|
|
||||||
public int K { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the query uses spatial search.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsSpatialSearch { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the center point for near queries.
|
|
||||||
/// </summary>
|
|
||||||
public (double Latitude, double Longitude) SpatialPoint { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the search radius in kilometers.
|
|
||||||
/// </summary>
|
|
||||||
public double RadiusKm { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the minimum point for within queries.
|
|
||||||
/// </summary>
|
|
||||||
public (double Latitude, double Longitude) SpatialMin { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the maximum point for within queries.
|
|
||||||
/// </summary>
|
|
||||||
public (double Latitude, double Longitude) SpatialMax { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the spatial query type.
|
|
||||||
/// </summary>
|
|
||||||
public SpatialQueryType SpatialType { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public enum SpatialQueryType { Near, Within }
|
/// <summary>
|
||||||
|
/// Attempts to optimize a query model using available indexes.
|
||||||
/// <summary>
|
/// </summary>
|
||||||
/// Attempts to optimize a query model using available indexes.
|
/// <typeparam name="T">The document type.</typeparam>
|
||||||
/// </summary>
|
/// <param name="model">The query model.</param>
|
||||||
/// <typeparam name="T">The document type.</typeparam>
|
/// <param name="indexes">The available collection indexes.</param>
|
||||||
/// <param name="model">The query model.</param>
|
/// <returns>An optimization result when optimization is possible; otherwise, <see langword="null" />.</returns>
|
||||||
/// <param name="indexes">The available collection indexes.</param>
|
public static OptimizationResult? TryOptimize<T>(QueryModel model, IEnumerable<CollectionIndexInfo> indexes)
|
||||||
/// <returns>An optimization result when optimization is possible; otherwise, <see langword="null"/>.</returns>
|
{
|
||||||
public static OptimizationResult? TryOptimize<T>(QueryModel model, IEnumerable<CollectionIndexInfo> indexes)
|
if (model.WhereClause == null) return null;
|
||||||
{
|
|
||||||
if (model.WhereClause == null) return null;
|
|
||||||
|
|
||||||
return OptimizeExpression(model.WhereClause.Body, model.WhereClause.Parameters[0], indexes);
|
return OptimizeExpression(model.WhereClause.Body, model.WhereClause.Parameters[0], indexes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter, IEnumerable<CollectionIndexInfo> indexes)
|
private static OptimizationResult? OptimizeExpression(Expression expression, ParameterExpression parameter,
|
||||||
|
IEnumerable<CollectionIndexInfo> indexes)
|
||||||
{
|
{
|
||||||
// ... (Existing AndAlso logic remains the same) ...
|
// ... (Existing AndAlso logic remains the same) ...
|
||||||
if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso)
|
if (expression is BinaryExpression binary && binary.NodeType == ExpressionType.AndAlso)
|
||||||
@@ -101,7 +35,6 @@ internal static class IndexOptimizer
|
|||||||
var right = OptimizeExpression(binary.Right, parameter, indexes);
|
var right = OptimizeExpression(binary.Right, parameter, indexes);
|
||||||
|
|
||||||
if (left != null && right != null && left.IndexName == right.IndexName)
|
if (left != null && right != null && left.IndexName == right.IndexName)
|
||||||
{
|
|
||||||
return new OptimizationResult
|
return new OptimizationResult
|
||||||
{
|
{
|
||||||
IndexName = left.IndexName,
|
IndexName = left.IndexName,
|
||||||
@@ -109,12 +42,11 @@ internal static class IndexOptimizer
|
|||||||
MaxValue = left.MaxValue ?? right.MaxValue,
|
MaxValue = left.MaxValue ?? right.MaxValue,
|
||||||
IsRange = true
|
IsRange = true
|
||||||
};
|
};
|
||||||
}
|
|
||||||
return left ?? right;
|
return left ?? right;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle Simple Binary Predicates
|
// Handle Simple Binary Predicates
|
||||||
var (propertyName, value, op) = ParseSimplePredicate(expression, parameter);
|
(string? propertyName, object? value, var op) = ParseSimplePredicate(expression, parameter);
|
||||||
if (propertyName != null)
|
if (propertyName != null)
|
||||||
{
|
{
|
||||||
var index = indexes.FirstOrDefault(i => Matches(i, propertyName));
|
var index = indexes.FirstOrDefault(i => Matches(i, propertyName));
|
||||||
@@ -128,55 +60,56 @@ internal static class IndexOptimizer
|
|||||||
result.MaxValue = value;
|
result.MaxValue = value;
|
||||||
result.IsRange = false;
|
result.IsRange = false;
|
||||||
break;
|
break;
|
||||||
case ExpressionType.GreaterThan:
|
case ExpressionType.GreaterThan:
|
||||||
case ExpressionType.GreaterThanOrEqual:
|
case ExpressionType.GreaterThanOrEqual:
|
||||||
result.MinValue = value;
|
result.MinValue = value;
|
||||||
result.MaxValue = null;
|
result.MaxValue = null;
|
||||||
result.IsRange = true;
|
result.IsRange = true;
|
||||||
break;
|
break;
|
||||||
case ExpressionType.LessThan:
|
case ExpressionType.LessThan:
|
||||||
case ExpressionType.LessThanOrEqual:
|
case ExpressionType.LessThanOrEqual:
|
||||||
result.MinValue = null;
|
result.MinValue = null;
|
||||||
result.MaxValue = value;
|
result.MaxValue = value;
|
||||||
result.IsRange = true;
|
result.IsRange = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Handle StartsWith
|
|
||||||
if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" && call.Object is MemberExpression member)
|
|
||||||
{
|
|
||||||
if (member.Expression == parameter && call.Arguments[0] is ConstantExpression constant && constant.Value is string prefix)
|
|
||||||
{
|
|
||||||
var index = indexes.FirstOrDefault(i => Matches(i, member.Member.Name));
|
|
||||||
if (index != null && index.Type == IndexType.BTree)
|
|
||||||
{
|
|
||||||
var nextPrefix = IncrementPrefix(prefix);
|
|
||||||
return new OptimizationResult
|
|
||||||
{
|
|
||||||
IndexName = index.Name,
|
|
||||||
MinValue = prefix,
|
|
||||||
MaxValue = nextPrefix,
|
|
||||||
IsRange = true
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle StartsWith
|
||||||
|
if (expression is MethodCallExpression call && call.Method.Name == "StartsWith" &&
|
||||||
|
call.Object is MemberExpression member)
|
||||||
|
if (member.Expression == parameter && call.Arguments[0] is ConstantExpression constant &&
|
||||||
|
constant.Value is string prefix)
|
||||||
|
{
|
||||||
|
var index = indexes.FirstOrDefault(i => Matches(i, member.Member.Name));
|
||||||
|
if (index != null && index.Type == IndexType.BTree)
|
||||||
|
{
|
||||||
|
string nextPrefix = IncrementPrefix(prefix);
|
||||||
|
return new OptimizationResult
|
||||||
|
{
|
||||||
|
IndexName = index.Name,
|
||||||
|
MinValue = prefix,
|
||||||
|
MaxValue = nextPrefix,
|
||||||
|
IsRange = true
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Handle Method Calls (VectorSearch, Near, Within)
|
// Handle Method Calls (VectorSearch, Near, Within)
|
||||||
if (expression is MethodCallExpression mcall)
|
if (expression is MethodCallExpression mcall)
|
||||||
{
|
{
|
||||||
// VectorSearch(this float[] vector, float[] query, int k)
|
// VectorSearch(this float[] vector, float[] query, int k)
|
||||||
if (mcall.Method.Name == "VectorSearch" && mcall.Arguments[0] is MemberExpression vMember && vMember.Expression == parameter)
|
if (mcall.Method.Name == "VectorSearch" && mcall.Arguments[0] is MemberExpression vMember &&
|
||||||
|
vMember.Expression == parameter)
|
||||||
{
|
{
|
||||||
var query = EvaluateExpression<float[]>(mcall.Arguments[1]);
|
float[] query = EvaluateExpression<float[]>(mcall.Arguments[1]);
|
||||||
var k = EvaluateExpression<int>(mcall.Arguments[2]);
|
var k = EvaluateExpression<int>(mcall.Arguments[2]);
|
||||||
|
|
||||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Vector && Matches(i, vMember.Member.Name));
|
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Vector && Matches(i, vMember.Member.Name));
|
||||||
if (index != null)
|
if (index != null)
|
||||||
{
|
|
||||||
return new OptimizationResult
|
return new OptimizationResult
|
||||||
{
|
{
|
||||||
IndexName = index.Name,
|
IndexName = index.Name,
|
||||||
@@ -184,18 +117,17 @@ internal static class IndexOptimizer
|
|||||||
VectorQuery = query,
|
VectorQuery = query,
|
||||||
K = k
|
K = k
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
// Near(this (double, double) point, (double, double) center, double radiusKm)
|
||||||
// Near(this (double, double) point, (double, double) center, double radiusKm)
|
if (mcall.Method.Name == "Near" && mcall.Arguments[0] is MemberExpression nMember &&
|
||||||
if (mcall.Method.Name == "Near" && mcall.Arguments[0] is MemberExpression nMember && nMember.Expression == parameter)
|
nMember.Expression == parameter)
|
||||||
{
|
{
|
||||||
var center = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
var center = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
||||||
var radius = EvaluateExpression<double>(mcall.Arguments[2]);
|
var radius = EvaluateExpression<double>(mcall.Arguments[2]);
|
||||||
|
|
||||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, nMember.Member.Name));
|
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, nMember.Member.Name));
|
||||||
if (index != null)
|
if (index != null)
|
||||||
{
|
|
||||||
return new OptimizationResult
|
return new OptimizationResult
|
||||||
{
|
{
|
||||||
IndexName = index.Name,
|
IndexName = index.Name,
|
||||||
@@ -204,18 +136,17 @@ internal static class IndexOptimizer
|
|||||||
SpatialPoint = center,
|
SpatialPoint = center,
|
||||||
RadiusKm = radius
|
RadiusKm = radius
|
||||||
};
|
};
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Within(this (double, double) point, (double, double) min, (double, double) max)
|
// Within(this (double, double) point, (double, double) min, (double, double) max)
|
||||||
if (mcall.Method.Name == "Within" && mcall.Arguments[0] is MemberExpression wMember && wMember.Expression == parameter)
|
if (mcall.Method.Name == "Within" && mcall.Arguments[0] is MemberExpression wMember &&
|
||||||
|
wMember.Expression == parameter)
|
||||||
{
|
{
|
||||||
var min = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
var min = EvaluateExpression<(double, double)>(mcall.Arguments[1]);
|
||||||
var max = EvaluateExpression<(double, double)>(mcall.Arguments[2]);
|
var max = EvaluateExpression<(double, double)>(mcall.Arguments[2]);
|
||||||
|
|
||||||
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, wMember.Member.Name));
|
var index = indexes.FirstOrDefault(i => i.Type == IndexType.Spatial && Matches(i, wMember.Member.Name));
|
||||||
if (index != null)
|
if (index != null)
|
||||||
{
|
|
||||||
return new OptimizationResult
|
return new OptimizationResult
|
||||||
{
|
{
|
||||||
IndexName = index.Name,
|
IndexName = index.Name,
|
||||||
@@ -224,7 +155,6 @@ internal static class IndexOptimizer
|
|||||||
SpatialMin = min,
|
SpatialMin = min,
|
||||||
SpatialMax = max
|
SpatialMax = max
|
||||||
};
|
};
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,10 +171,7 @@ internal static class IndexOptimizer
|
|||||||
|
|
||||||
private static T EvaluateExpression<T>(Expression expression)
|
private static T EvaluateExpression<T>(Expression expression)
|
||||||
{
|
{
|
||||||
if (expression is ConstantExpression constant)
|
if (expression is ConstantExpression constant) return (T)constant.Value!;
|
||||||
{
|
|
||||||
return (T)constant.Value!;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evaluate more complex expressions (closures, properties, etc.)
|
// Evaluate more complex expressions (closures, properties, etc.)
|
||||||
var lambda = Expression.Lambda(expression);
|
var lambda = Expression.Lambda(expression);
|
||||||
@@ -258,7 +185,8 @@ internal static class IndexOptimizer
|
|||||||
return index.PropertyPaths[0].Equals(propertyName, StringComparison.OrdinalIgnoreCase);
|
return index.PropertyPaths[0].Equals(propertyName, StringComparison.OrdinalIgnoreCase);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression, ParameterExpression parameter)
|
private static (string? propertyName, object? value, ExpressionType op) ParseSimplePredicate(Expression expression,
|
||||||
|
ParameterExpression parameter)
|
||||||
{
|
{
|
||||||
if (expression is BinaryExpression binary)
|
if (expression is BinaryExpression binary)
|
||||||
{
|
{
|
||||||
@@ -273,27 +201,99 @@ internal static class IndexOptimizer
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (left is MemberExpression member && right is ConstantExpression constant)
|
if (left is MemberExpression member && right is ConstantExpression constant)
|
||||||
{
|
|
||||||
if (member.Expression == parameter)
|
if (member.Expression == parameter)
|
||||||
return (member.Member.Name, constant.Value, nodeType);
|
return (member.Member.Name, constant.Value, nodeType);
|
||||||
}
|
|
||||||
|
// Handle Convert
|
||||||
// Handle Convert
|
if (left is UnaryExpression unary && unary.Operand is MemberExpression member2 &&
|
||||||
if (left is UnaryExpression unary && unary.Operand is MemberExpression member2 && right is ConstantExpression constant2)
|
right is ConstantExpression constant2)
|
||||||
{
|
|
||||||
if (member2.Expression == parameter)
|
if (member2.Expression == parameter)
|
||||||
return (member2.Member.Name, constant2.Value, nodeType);
|
return (member2.Member.Name, constant2.Value, nodeType);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return (null, null, ExpressionType.Default);
|
return (null, null, ExpressionType.Default);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ExpressionType Flip(ExpressionType type) => type switch
|
private static ExpressionType Flip(ExpressionType type)
|
||||||
{
|
{
|
||||||
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
return type switch
|
||||||
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
{
|
||||||
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
ExpressionType.GreaterThan => ExpressionType.LessThan,
|
||||||
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
ExpressionType.LessThan => ExpressionType.GreaterThan,
|
||||||
_ => type
|
ExpressionType.GreaterThanOrEqual => ExpressionType.LessThanOrEqual,
|
||||||
};
|
ExpressionType.LessThanOrEqual => ExpressionType.GreaterThanOrEqual,
|
||||||
}
|
_ => type
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Represents the selected index and bounds for an optimized query.
|
||||||
|
/// </summary>
|
||||||
|
public class OptimizationResult
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the selected index name.
|
||||||
|
/// </summary>
|
||||||
|
public string IndexName { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the minimum bound value.
|
||||||
|
/// </summary>
|
||||||
|
public object? MinValue { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the maximum bound value.
|
||||||
|
/// </summary>
|
||||||
|
public object? MaxValue { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the query uses a range.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsRange { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the query uses vector search.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsVectorSearch { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the vector query values.
|
||||||
|
/// </summary>
|
||||||
|
public float[]? VectorQuery { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the number of nearest neighbors for vector search.
|
||||||
|
/// </summary>
|
||||||
|
public int K { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the query uses spatial search.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsSpatialSearch { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the center point for near queries.
|
||||||
|
/// </summary>
|
||||||
|
public (double Latitude, double Longitude) SpatialPoint { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the search radius in kilometers.
|
||||||
|
/// </summary>
|
||||||
|
public double RadiusKm { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the minimum point for within queries.
|
||||||
|
/// </summary>
|
||||||
|
public (double Latitude, double Longitude) SpatialMin { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the maximum point for within queries.
|
||||||
|
/// </summary>
|
||||||
|
public (double Latitude, double Longitude) SpatialMax { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the spatial query type.
|
||||||
|
/// </summary>
|
||||||
|
public SpatialQueryType SpatialType { get; set; }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,36 +1,36 @@
|
|||||||
using System.Linq.Expressions;
|
using System.Linq.Expressions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Query;
|
namespace ZB.MOM.WW.CBDD.Core.Query;
|
||||||
|
|
||||||
internal class QueryModel
|
internal class QueryModel
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the filter expression.
|
/// Gets or sets the filter expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public LambdaExpression? WhereClause { get; set; }
|
public LambdaExpression? WhereClause { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the projection expression.
|
/// Gets or sets the projection expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public LambdaExpression? SelectClause { get; set; }
|
public LambdaExpression? SelectClause { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the ordering expression.
|
/// Gets or sets the ordering expression.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public LambdaExpression? OrderByClause { get; set; }
|
public LambdaExpression? OrderByClause { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the maximum number of results to return.
|
/// Gets or sets the maximum number of results to return.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int? Take { get; set; }
|
public int? Take { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the number of results to skip.
|
/// Gets or sets the number of results to skip.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int? Skip { get; set; }
|
public int? Skip { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets a value indicating whether ordering is descending.
|
/// Gets or sets a value indicating whether ordering is descending.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool OrderDescending { get; set; }
|
public bool OrderDescending { get; set; }
|
||||||
}
|
}
|
||||||
@@ -1,13 +1,13 @@
|
|||||||
using System.Runtime.InteropServices;
|
using System.Buffers;
|
||||||
|
using System.Buffers.Binary;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using ZB.MOM.WW.CBDD.Core;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Page for storing dictionary entries (Key -> Value map).
|
/// Page for storing dictionary entries (Key -> Value map).
|
||||||
/// Uses a sorted list of keys for binary search within the page.
|
/// Uses a sorted list of keys for binary search within the page.
|
||||||
/// Supports chaining via PageHeader.NextPageId for dictionaries larger than one page.
|
/// Supports chaining via PageHeader.NextPageId for dictionaries larger than one page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public struct DictionaryPage
|
public struct DictionaryPage
|
||||||
{
|
{
|
||||||
@@ -25,16 +25,16 @@ public struct DictionaryPage
|
|||||||
private const int OffsetsStart = 36;
|
private const int OffsetsStart = 36;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Values 0-100 are reserved for internal system keys (e.g. _id, _v).
|
/// Values 0-100 are reserved for internal system keys (e.g. _id, _v).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public const ushort ReservedValuesEnd = 100;
|
public const ushort ReservedValuesEnd = 100;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initialize a new dictionary page
|
/// Initialize a new dictionary page
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer to initialize.</param>
|
/// <param name="page">The page buffer to initialize.</param>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
public static void Initialize(Span<byte> page, uint pageId)
|
public static void Initialize(Span<byte> page, uint pageId)
|
||||||
{
|
{
|
||||||
// 1. Write Page Header
|
// 1. Write Page Header
|
||||||
var header = new PageHeader
|
var header = new PageHeader
|
||||||
@@ -49,43 +49,40 @@ public struct DictionaryPage
|
|||||||
header.WriteTo(page);
|
header.WriteTo(page);
|
||||||
|
|
||||||
// 2. Initialize Counts
|
// 2. Initialize Counts
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), 0);
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), 0);
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), (ushort)page.Length);
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), (ushort)page.Length);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Inserts a key-value pair into the page.
|
/// Inserts a key-value pair into the page.
|
||||||
/// Returns false if there is not enough space.
|
/// Returns false if there is not enough space.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="key">The dictionary key.</param>
|
/// <param name="key">The dictionary key.</param>
|
||||||
/// <param name="value">The value mapped to the key.</param>
|
/// <param name="value">The value mapped to the key.</param>
|
||||||
/// <returns><see langword="true"/> if the entry was inserted; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the entry was inserted; otherwise, <see langword="false" />.</returns>
|
||||||
public static bool Insert(Span<byte> page, string key, ushort value)
|
public static bool Insert(Span<byte> page, string key, ushort value)
|
||||||
{
|
{
|
||||||
var keyByteCount = Encoding.UTF8.GetByteCount(key);
|
int keyByteCount = Encoding.UTF8.GetByteCount(key);
|
||||||
if (keyByteCount > 255) throw new ArgumentException("Key length must be <= 255 bytes");
|
if (keyByteCount > 255) throw new ArgumentException("Key length must be <= 255 bytes");
|
||||||
|
|
||||||
// Entry Size: KeyLen(1) + Key(N) + Value(2)
|
// Entry Size: KeyLen(1) + Key(N) + Value(2)
|
||||||
var entrySize = 1 + keyByteCount + 2;
|
int entrySize = 1 + keyByteCount + 2;
|
||||||
var requiredSpace = entrySize + 2; // +2 for Offset entry
|
int requiredSpace = entrySize + 2; // +2 for Offset entry
|
||||||
|
|
||||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
ushort count = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||||
var freeSpaceEnd = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(FreeSpaceEndOffset));
|
ushort freeSpaceEnd = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(FreeSpaceEndOffset));
|
||||||
|
|
||||||
var offsetsEnd = OffsetsStart + (count * 2);
|
|
||||||
var freeSpace = freeSpaceEnd - offsetsEnd;
|
|
||||||
|
|
||||||
if (freeSpace < requiredSpace)
|
int offsetsEnd = OffsetsStart + count * 2;
|
||||||
{
|
int freeSpace = freeSpaceEnd - offsetsEnd;
|
||||||
return false; // Page Full
|
|
||||||
}
|
if (freeSpace < requiredSpace) return false; // Page Full
|
||||||
|
|
||||||
// 1. Prepare Data
|
// 1. Prepare Data
|
||||||
var insertionOffset = (ushort)(freeSpaceEnd - entrySize);
|
var insertionOffset = (ushort)(freeSpaceEnd - entrySize);
|
||||||
page[insertionOffset] = (byte)keyByteCount; // Write Key Length
|
page[insertionOffset] = (byte)keyByteCount; // Write Key Length
|
||||||
Encoding.UTF8.GetBytes(key, page.Slice(insertionOffset + 1, keyByteCount)); // Write Key
|
Encoding.UTF8.GetBytes(key, page.Slice(insertionOffset + 1, keyByteCount)); // Write Key
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(insertionOffset + 1 + keyByteCount), value); // Write Value
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(insertionOffset + 1 + keyByteCount), value); // Write Value
|
||||||
|
|
||||||
// 2. Insert Offset into Sorted List
|
// 2. Insert Offset into Sorted List
|
||||||
// Find insert Index using spans
|
// Find insert Index using spans
|
||||||
@@ -95,57 +92,57 @@ public struct DictionaryPage
|
|||||||
// Shift offsets if needed
|
// Shift offsets if needed
|
||||||
if (insertIndex < count)
|
if (insertIndex < count)
|
||||||
{
|
{
|
||||||
var src = page.Slice(OffsetsStart + (insertIndex * 2), (count - insertIndex) * 2);
|
var src = page.Slice(OffsetsStart + insertIndex * 2, (count - insertIndex) * 2);
|
||||||
var dest = page.Slice(OffsetsStart + ((insertIndex + 1) * 2));
|
var dest = page.Slice(OffsetsStart + (insertIndex + 1) * 2);
|
||||||
src.CopyTo(dest);
|
src.CopyTo(dest);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write new offset
|
// Write new offset
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(OffsetsStart + (insertIndex * 2)), insertionOffset);
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(OffsetsStart + insertIndex * 2), insertionOffset);
|
||||||
|
|
||||||
// 3. Update Metadata
|
// 3. Update Metadata
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), (ushort)(count + 1));
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(CountOffset), (ushort)(count + 1));
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), insertionOffset);
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(FreeSpaceEndOffset), insertionOffset);
|
||||||
|
|
||||||
// Update FreeBytes in header (approximate)
|
// Update FreeBytes in header (approximate)
|
||||||
var pageHeader = PageHeader.ReadFrom(page);
|
var pageHeader = PageHeader.ReadFrom(page);
|
||||||
pageHeader.FreeBytes = (ushort)(insertionOffset - (OffsetsStart + ((count + 1) * 2)));
|
pageHeader.FreeBytes = (ushort)(insertionOffset - (OffsetsStart + (count + 1) * 2));
|
||||||
pageHeader.WriteTo(page);
|
pageHeader.WriteTo(page);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Tries to find a value for the given key in THIS page.
|
/// Tries to find a value for the given key in THIS page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="keyBytes">The UTF-8 encoded key bytes.</param>
|
/// <param name="keyBytes">The UTF-8 encoded key bytes.</param>
|
||||||
/// <param name="value">When this method returns, contains the found value.</param>
|
/// <param name="value">When this method returns, contains the found value.</param>
|
||||||
/// <returns><see langword="true"/> if the key was found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the key was found; otherwise, <see langword="false" />.</returns>
|
||||||
public static bool TryFind(ReadOnlySpan<byte> page, ReadOnlySpan<byte> keyBytes, out ushort value)
|
public static bool TryFind(ReadOnlySpan<byte> page, ReadOnlySpan<byte> keyBytes, out ushort value)
|
||||||
{
|
{
|
||||||
value = 0;
|
value = 0;
|
||||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
ushort count = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||||
if (count == 0) return false;
|
if (count == 0) return false;
|
||||||
|
|
||||||
// Binary Search
|
// Binary Search
|
||||||
int low = 0;
|
var low = 0;
|
||||||
int high = count - 1;
|
int high = count - 1;
|
||||||
|
|
||||||
while (low <= high)
|
while (low <= high)
|
||||||
{
|
{
|
||||||
int mid = low + (high - low) / 2;
|
int mid = low + (high - low) / 2;
|
||||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (mid * 2)));
|
ushort offset = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + mid * 2));
|
||||||
|
|
||||||
// Read Key at Offset
|
// Read Key at Offset
|
||||||
var keyLen = page[offset];
|
byte keyLen = page[offset];
|
||||||
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
||||||
|
|
||||||
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
||||||
|
|
||||||
if (comparison == 0)
|
if (comparison == 0)
|
||||||
{
|
{
|
||||||
value = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
value = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,126 +155,125 @@ public struct DictionaryPage
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Tries to find a value for the given key across a chain of DictionaryPages.
|
/// Tries to find a value for the given key across a chain of DictionaryPages.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="storage">The storage engine used to read pages.</param>
|
/// <param name="storage">The storage engine used to read pages.</param>
|
||||||
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
||||||
/// <param name="key">The key to search for.</param>
|
/// <param name="key">The key to search for.</param>
|
||||||
/// <param name="value">When this method returns, contains the found value.</param>
|
/// <param name="value">When this method returns, contains the found value.</param>
|
||||||
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
||||||
/// <returns><see langword="true"/> if the key was found; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the key was found; otherwise, <see langword="false" />.</returns>
|
||||||
public static bool TryFindGlobal(StorageEngine storage, uint startPageId, string key, out ushort value, ulong? transactionId = null)
|
public static bool TryFindGlobal(StorageEngine storage, uint startPageId, string key, out ushort value,
|
||||||
|
ulong? transactionId = null)
|
||||||
{
|
{
|
||||||
var keyByteCount = Encoding.UTF8.GetByteCount(key);
|
int keyByteCount = Encoding.UTF8.GetByteCount(key);
|
||||||
Span<byte> keyBytes = keyByteCount <= 256 ? stackalloc byte[keyByteCount] : new byte[keyByteCount];
|
var keyBytes = keyByteCount <= 256 ? stackalloc byte[keyByteCount] : new byte[keyByteCount];
|
||||||
Encoding.UTF8.GetBytes(key, keyBytes);
|
Encoding.UTF8.GetBytes(key, keyBytes);
|
||||||
|
|
||||||
var pageId = startPageId;
|
uint pageId = startPageId;
|
||||||
var pageBuffer = System.Buffers.ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
byte[] pageBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
while (pageId != 0)
|
while (pageId != 0)
|
||||||
{
|
{
|
||||||
// Read page
|
// Read page
|
||||||
storage.ReadPage(pageId, transactionId, pageBuffer);
|
storage.ReadPage(pageId, transactionId, pageBuffer);
|
||||||
|
|
||||||
// TryFind in this page
|
// TryFind in this page
|
||||||
if (TryFind(pageBuffer, keyBytes, out value))
|
if (TryFind(pageBuffer, keyBytes, out value)) return true;
|
||||||
{
|
|
||||||
return true;
|
// Move to next page
|
||||||
}
|
var header = PageHeader.ReadFrom(pageBuffer);
|
||||||
|
|
||||||
// Move to next page
|
|
||||||
var header = PageHeader.ReadFrom(pageBuffer);
|
|
||||||
pageId = header.NextPageId;
|
pageId = header.NextPageId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
System.Buffers.ArrayPool<byte>.Shared.Return(pageBuffer);
|
ArrayPool<byte>.Shared.Return(pageBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
value = 0;
|
value = 0;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int FindInsertIndex(ReadOnlySpan<byte> page, int count, ReadOnlySpan<byte> keyBytes)
|
private static int FindInsertIndex(ReadOnlySpan<byte> page, int count, ReadOnlySpan<byte> keyBytes)
|
||||||
{
|
{
|
||||||
int low = 0;
|
var low = 0;
|
||||||
int high = count - 1;
|
int high = count - 1;
|
||||||
|
|
||||||
while (low <= high)
|
while (low <= high)
|
||||||
{
|
{
|
||||||
int mid = low + (high - low) / 2;
|
int mid = low + (high - low) / 2;
|
||||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (mid * 2)));
|
ushort offset = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + mid * 2));
|
||||||
|
|
||||||
var keyLen = page[offset];
|
byte keyLen = page[offset];
|
||||||
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
var entryKeySpan = page.Slice(offset + 1, keyLen);
|
||||||
|
|
||||||
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
int comparison = entryKeySpan.SequenceCompareTo(keyBytes);
|
||||||
|
|
||||||
if (comparison == 0) return mid;
|
if (comparison == 0) return mid;
|
||||||
if (comparison < 0)
|
if (comparison < 0)
|
||||||
low = mid + 1;
|
low = mid + 1;
|
||||||
else
|
else
|
||||||
high = mid - 1;
|
high = mid - 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return low;
|
return low;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets all entries in the page (for debugging/dumping)
|
/// Gets all entries in the page (for debugging/dumping)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns>All key-value pairs in the page.</returns>
|
/// <returns>All key-value pairs in the page.</returns>
|
||||||
public static IEnumerable<(string Key, ushort Value)> GetAll(ReadOnlySpan<byte> page)
|
public static IEnumerable<(string Key, ushort Value)> GetAll(ReadOnlySpan<byte> page)
|
||||||
{
|
{
|
||||||
var count = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
ushort count = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(CountOffset));
|
||||||
var list = new List<(string Key, ushort Value)>();
|
var list = new List<(string Key, ushort Value)>();
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
var offset = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + (i * 2)));
|
ushort offset = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(OffsetsStart + i * 2));
|
||||||
var keyLen = page[offset];
|
byte keyLen = page[offset];
|
||||||
var keyStr = Encoding.UTF8.GetString(page.Slice(offset + 1, keyLen));
|
string keyStr = Encoding.UTF8.GetString(page.Slice(offset + 1, keyLen));
|
||||||
var val = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
ushort val = BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(offset + 1 + keyLen));
|
||||||
list.Add((keyStr, val));
|
list.Add((keyStr, val));
|
||||||
}
|
}
|
||||||
|
|
||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
/// <summary>
|
|
||||||
/// Retrieves all key-value pairs across a chain of DictionaryPages.
|
/// <summary>
|
||||||
/// Used for rebuilding the in-memory cache.
|
/// Retrieves all key-value pairs across a chain of DictionaryPages.
|
||||||
/// </summary>
|
/// Used for rebuilding the in-memory cache.
|
||||||
/// <param name="storage">The storage engine used to read pages.</param>
|
/// </summary>
|
||||||
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
/// <param name="storage">The storage engine used to read pages.</param>
|
||||||
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
/// <param name="startPageId">The first page in the dictionary chain.</param>
|
||||||
/// <returns>All key-value pairs across the page chain.</returns>
|
/// <param name="transactionId">Optional transaction identifier for isolated reads.</param>
|
||||||
public static IEnumerable<(string Key, ushort Value)> FindAllGlobal(StorageEngine storage, uint startPageId, ulong? transactionId = null)
|
/// <returns>All key-value pairs across the page chain.</returns>
|
||||||
|
public static IEnumerable<(string Key, ushort Value)> FindAllGlobal(StorageEngine storage, uint startPageId,
|
||||||
|
ulong? transactionId = null)
|
||||||
{
|
{
|
||||||
var pageId = startPageId;
|
uint pageId = startPageId;
|
||||||
var pageBuffer = System.Buffers.ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
byte[] pageBuffer = ArrayPool<byte>.Shared.Rent(storage.PageSize);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
while (pageId != 0)
|
while (pageId != 0)
|
||||||
{
|
{
|
||||||
// Read page
|
// Read page
|
||||||
storage.ReadPage(pageId, transactionId, pageBuffer);
|
storage.ReadPage(pageId, transactionId, pageBuffer);
|
||||||
|
|
||||||
// Get all entries in this page
|
// Get all entries in this page
|
||||||
foreach (var entry in GetAll(pageBuffer))
|
foreach (var entry in GetAll(pageBuffer)) yield return entry;
|
||||||
{
|
|
||||||
yield return entry;
|
// Move to next page
|
||||||
}
|
var header = PageHeader.ReadFrom(pageBuffer);
|
||||||
|
|
||||||
// Move to next page
|
|
||||||
var header = PageHeader.ReadFrom(pageBuffer);
|
|
||||||
pageId = header.NextPageId;
|
pageId = header.NextPageId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
System.Buffers.ArrayPool<byte>.Shared.Return(pageBuffer);
|
ArrayPool<byte>.Shared.Return(pageBuffer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,41 +1,46 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Narrow storage port for index structures (page operations + allocation only).
|
/// Narrow storage port for index structures (page operations + allocation only).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal interface IIndexStorage
|
internal interface IIndexStorage
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the PageSize.
|
/// Gets or sets the PageSize.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
int PageSize { get; }
|
int PageSize { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes AllocatePage.
|
/// Executes AllocatePage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
uint AllocatePage();
|
uint AllocatePage();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes FreePage.
|
/// Executes FreePage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
void FreePage(uint pageId);
|
void FreePage(uint pageId);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes ReadPage.
|
/// Executes ReadPage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
/// <param name="transactionId">The optional transaction identifier.</param>
|
/// <param name="transactionId">The optional transaction identifier.</param>
|
||||||
/// <param name="destination">The destination buffer.</param>
|
/// <param name="destination">The destination buffer.</param>
|
||||||
void ReadPage(uint pageId, ulong? transactionId, Span<byte> destination);
|
void ReadPage(uint pageId, ulong? transactionId, Span<byte> destination);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes WritePage.
|
/// Executes WritePage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
/// <param name="transactionId">The transaction identifier.</param>
|
/// <param name="transactionId">The transaction identifier.</param>
|
||||||
/// <param name="data">The source page data.</param>
|
/// <param name="data">The source page data.</param>
|
||||||
void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data);
|
void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Executes WritePageImmediate.
|
/// Executes WritePageImmediate.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
/// <param name="data">The source page data.</param>
|
/// <param name="data">The source page data.</param>
|
||||||
void WritePageImmediate(uint pageId, ReadOnlySpan<byte> data);
|
void WritePageImmediate(uint pageId, ReadOnlySpan<byte> data);
|
||||||
}
|
}
|
||||||
@@ -8,111 +8,112 @@ using ZB.MOM.WW.CBDD.Core.Transactions;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Storage port used by collection/index orchestration to avoid concrete engine coupling.
|
/// Storage port used by collection/index orchestration to avoid concrete engine coupling.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal interface IStorageEngine : IIndexStorage, IDisposable
|
internal interface IStorageEngine : IIndexStorage, IDisposable
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current page count.
|
/// Gets the current page count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
uint PageCount { get; }
|
uint PageCount { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the active change stream dispatcher.
|
/// Gets the active change stream dispatcher.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
ChangeStreamDispatcher? Cdc { get; }
|
ChangeStreamDispatcher? Cdc { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets compression options used by the storage engine.
|
/// Gets compression options used by the storage engine.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
CompressionOptions CompressionOptions { get; }
|
CompressionOptions CompressionOptions { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the compression service.
|
/// Gets the compression service.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
CompressionService CompressionService { get; }
|
CompressionService CompressionService { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets compression telemetry for the storage engine.
|
/// Gets compression telemetry for the storage engine.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
CompressionTelemetry CompressionTelemetry { get; }
|
CompressionTelemetry CompressionTelemetry { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Determines whether a page is locked.
|
/// Determines whether a page is locked.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier to inspect.</param>
|
/// <param name="pageId">The page identifier to inspect.</param>
|
||||||
/// <param name="excludingTxId">A transaction identifier to exclude from lock checks.</param>
|
/// <param name="excludingTxId">A transaction identifier to exclude from lock checks.</param>
|
||||||
bool IsPageLocked(uint pageId, ulong excludingTxId);
|
bool IsPageLocked(uint pageId, ulong excludingTxId);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Registers the change stream dispatcher.
|
/// Registers the change stream dispatcher.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="cdc">The change stream dispatcher instance.</param>
|
/// <param name="cdc">The change stream dispatcher instance.</param>
|
||||||
void RegisterCdc(ChangeStreamDispatcher cdc);
|
void RegisterCdc(ChangeStreamDispatcher cdc);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins a transaction.
|
/// Begins a transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||||
Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted);
|
Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins a transaction asynchronously.
|
/// Begins a transaction asynchronously.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||||
/// <param name="ct">A cancellation token.</param>
|
/// <param name="ct">A cancellation token.</param>
|
||||||
Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, CancellationToken ct = default);
|
Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted,
|
||||||
|
CancellationToken ct = default);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets collection metadata by name.
|
/// Gets collection metadata by name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The collection name.</param>
|
/// <param name="name">The collection name.</param>
|
||||||
CollectionMetadata? GetCollectionMetadata(string name);
|
CollectionMetadata? GetCollectionMetadata(string name);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Saves collection metadata.
|
/// Saves collection metadata.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="metadata">The metadata to persist.</param>
|
/// <param name="metadata">The metadata to persist.</param>
|
||||||
void SaveCollectionMetadata(CollectionMetadata metadata);
|
void SaveCollectionMetadata(CollectionMetadata metadata);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Registers document mappers.
|
/// Registers document mappers.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="mappers">The mapper instances to register.</param>
|
/// <param name="mappers">The mapper instances to register.</param>
|
||||||
void RegisterMappers(IEnumerable<IDocumentMapper> mappers);
|
void RegisterMappers(IEnumerable<IDocumentMapper> mappers);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets schema chain entries for the specified root page.
|
/// Gets schema chain entries for the specified root page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rootPageId">The schema root page identifier.</param>
|
/// <param name="rootPageId">The schema root page identifier.</param>
|
||||||
List<BsonSchema> GetSchemas(uint rootPageId);
|
List<BsonSchema> GetSchemas(uint rootPageId);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Appends a schema to the specified schema chain.
|
/// Appends a schema to the specified schema chain.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rootPageId">The schema root page identifier.</param>
|
/// <param name="rootPageId">The schema root page identifier.</param>
|
||||||
/// <param name="schema">The schema to append.</param>
|
/// <param name="schema">The schema to append.</param>
|
||||||
uint AppendSchema(uint rootPageId, BsonSchema schema);
|
uint AppendSchema(uint rootPageId, BsonSchema schema);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the key-to-token mapping.
|
/// Gets the key-to-token mapping.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
ConcurrentDictionary<string, ushort> GetKeyMap();
|
ConcurrentDictionary<string, ushort> GetKeyMap();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the token-to-key mapping.
|
/// Gets the token-to-key mapping.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
ConcurrentDictionary<ushort, string> GetKeyReverseMap();
|
ConcurrentDictionary<ushort, string> GetKeyReverseMap();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or creates a dictionary token for the specified key.
|
/// Gets or creates a dictionary token for the specified key.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="key">The key value.</param>
|
/// <param name="key">The key value.</param>
|
||||||
ushort GetOrAddDictionaryEntry(string key);
|
ushort GetOrAddDictionaryEntry(string key);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Registers key values in the dictionary mapping.
|
/// Registers key values in the dictionary mapping.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="keys">The keys to register.</param>
|
/// <param name="keys">The keys to register.</param>
|
||||||
void RegisterKeys(IEnumerable<string> keys);
|
void RegisterKeys(IEnumerable<string> keys);
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -3,57 +3,45 @@ using System.Runtime.InteropServices;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a page header in the database file.
|
/// Represents a page header in the database file.
|
||||||
/// Fixed 32-byte structure at the start of each page.
|
/// Fixed 32-byte structure at the start of each page.
|
||||||
/// Implemented as struct for efficient memory layout.
|
/// Implemented as struct for efficient memory layout.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||||
public struct PageHeader
|
public struct PageHeader
|
||||||
{
|
{
|
||||||
/// <summary>Page ID (offset in pages from start of file)</summary>
|
/// <summary>Page ID (offset in pages from start of file)</summary>
|
||||||
[FieldOffset(0)]
|
[FieldOffset(0)] public uint PageId;
|
||||||
public uint PageId;
|
|
||||||
|
|
||||||
/// <summary>Type of this page</summary>
|
/// <summary>Type of this page</summary>
|
||||||
[FieldOffset(4)]
|
[FieldOffset(4)] public PageType PageType;
|
||||||
public PageType PageType;
|
|
||||||
|
|
||||||
/// <summary>Number of free bytes in this page</summary>
|
/// <summary>Number of free bytes in this page</summary>
|
||||||
[FieldOffset(5)]
|
[FieldOffset(5)] public ushort FreeBytes;
|
||||||
public ushort FreeBytes;
|
|
||||||
|
|
||||||
/// <summary>ID of next page in linked list (0 if none). For Page 0 (Header), this points to the First Free Page.</summary>
|
/// <summary>ID of next page in linked list (0 if none). For Page 0 (Header), this points to the First Free Page.</summary>
|
||||||
[FieldOffset(7)]
|
[FieldOffset(7)] public uint NextPageId;
|
||||||
public uint NextPageId;
|
|
||||||
|
|
||||||
/// <summary>Transaction ID that last modified this page</summary>
|
/// <summary>Transaction ID that last modified this page</summary>
|
||||||
[FieldOffset(11)]
|
[FieldOffset(11)] public ulong TransactionId;
|
||||||
public ulong TransactionId;
|
|
||||||
|
|
||||||
/// <summary>Checksum for data integrity (CRC32)</summary>
|
/// <summary>Checksum for data integrity (CRC32)</summary>
|
||||||
[FieldOffset(19)]
|
[FieldOffset(19)] public uint Checksum;
|
||||||
public uint Checksum;
|
|
||||||
|
|
||||||
/// <summary>Dictionary Root Page ID (Only used in Page 0 / File Header)</summary>
|
/// <summary>Dictionary Root Page ID (Only used in Page 0 / File Header)</summary>
|
||||||
[FieldOffset(23)]
|
[FieldOffset(23)] public uint DictionaryRootPageId;
|
||||||
public uint DictionaryRootPageId;
|
|
||||||
|
|
||||||
[FieldOffset(27)]
|
[FieldOffset(27)] private byte _reserved5;
|
||||||
private byte _reserved5;
|
[FieldOffset(28)] private byte _reserved6;
|
||||||
[FieldOffset(28)]
|
[FieldOffset(29)] private byte _reserved7;
|
||||||
private byte _reserved6;
|
[FieldOffset(30)] private byte _reserved8;
|
||||||
[FieldOffset(29)]
|
[FieldOffset(31)] private byte _reserved9;
|
||||||
private byte _reserved7;
|
|
||||||
[FieldOffset(30)]
|
|
||||||
private byte _reserved8;
|
|
||||||
[FieldOffset(31)]
|
|
||||||
private byte _reserved9;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes the header to a span
|
/// Writes the header to a span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span that receives the serialized header.</param>
|
/// <param name="destination">The destination span that receives the serialized header.</param>
|
||||||
public readonly void WriteTo(Span<byte> destination)
|
public readonly void WriteTo(Span<byte> destination)
|
||||||
{
|
{
|
||||||
if (destination.Length < 32)
|
if (destination.Length < 32)
|
||||||
throw new ArgumentException("Destination must be at least 32 bytes");
|
throw new ArgumentException("Destination must be at least 32 bytes");
|
||||||
@@ -61,15 +49,15 @@ public struct PageHeader
|
|||||||
MemoryMarshal.Write(destination, in this);
|
MemoryMarshal.Write(destination, in this);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a header from a span
|
/// Reads a header from a span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The source span containing a serialized header.</param>
|
/// <param name="source">The source span containing a serialized header.</param>
|
||||||
public static PageHeader ReadFrom(ReadOnlySpan<byte> source)
|
public static PageHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||||
{
|
{
|
||||||
if (source.Length < 32)
|
if (source.Length < 32)
|
||||||
throw new ArgumentException("Source must be at least 32 bytes");
|
throw new ArgumentException("Source must be at least 32 bytes");
|
||||||
|
|
||||||
return MemoryMarshal.Read<PageHeader>(source);
|
return MemoryMarshal.Read<PageHeader>(source);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,28 +1,28 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Page types in the database file
|
/// Page types in the database file
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum PageType : byte
|
public enum PageType : byte
|
||||||
{
|
{
|
||||||
/// <summary>Empty/free page</summary>
|
/// <summary>Empty/free page</summary>
|
||||||
Empty = 0,
|
Empty = 0,
|
||||||
|
|
||||||
/// <summary>File header page (page 0)</summary>
|
/// <summary>File header page (page 0)</summary>
|
||||||
Header = 1,
|
Header = 1,
|
||||||
|
|
||||||
/// <summary>Collection metadata page</summary>
|
/// <summary>Collection metadata page</summary>
|
||||||
Collection = 2,
|
Collection = 2,
|
||||||
|
|
||||||
/// <summary>Data page containing documents</summary>
|
/// <summary>Data page containing documents</summary>
|
||||||
Data = 3,
|
Data = 3,
|
||||||
|
|
||||||
/// <summary>Index B+Tree node page</summary>
|
/// <summary>Index B+Tree node page</summary>
|
||||||
Index = 4,
|
Index = 4,
|
||||||
|
|
||||||
/// <summary>Free page list</summary>
|
/// <summary>Free page list</summary>
|
||||||
FreeList = 5,
|
FreeList = 5,
|
||||||
|
|
||||||
/// <summary>Overflow page for large documents</summary>
|
/// <summary>Overflow page for large documents</summary>
|
||||||
Overflow = 6,
|
Overflow = 6,
|
||||||
|
|
||||||
@@ -40,4 +40,4 @@ public enum PageType : byte
|
|||||||
|
|
||||||
/// <summary>GEO Spatial index page</summary>
|
/// <summary>GEO Spatial index page</summary>
|
||||||
Spatial = 11
|
Spatial = 11
|
||||||
}
|
}
|
||||||
@@ -1,50 +1,43 @@
|
|||||||
using System.Runtime.InteropServices;
|
using System.Buffers.Binary;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
/// <summary>
|
|
||||||
/// Header for slotted pages supporting multiple variable-size documents per page.
|
/// <summary>
|
||||||
/// Fixed 24-byte structure at start of each data page.
|
/// Header for slotted pages supporting multiple variable-size documents per page.
|
||||||
/// </summary>
|
/// Fixed 24-byte structure at start of each data page.
|
||||||
[StructLayout(LayoutKind.Explicit, Size = 24)]
|
/// </summary>
|
||||||
|
[StructLayout(LayoutKind.Explicit, Size = 24)]
|
||||||
public struct SlottedPageHeader
|
public struct SlottedPageHeader
|
||||||
{
|
{
|
||||||
/// <summary>Page ID</summary>
|
/// <summary>Page ID</summary>
|
||||||
[FieldOffset(0)]
|
[FieldOffset(0)] public uint PageId;
|
||||||
public uint PageId;
|
|
||||||
|
|
||||||
/// <summary>Type of page (Data, Overflow, Index, Metadata)</summary>
|
/// <summary>Type of page (Data, Overflow, Index, Metadata)</summary>
|
||||||
[FieldOffset(4)]
|
[FieldOffset(4)] public PageType PageType;
|
||||||
public PageType PageType;
|
|
||||||
|
|
||||||
/// <summary>Number of slot entries in this page</summary>
|
/// <summary>Number of slot entries in this page</summary>
|
||||||
[FieldOffset(8)]
|
[FieldOffset(8)] public ushort SlotCount;
|
||||||
public ushort SlotCount;
|
|
||||||
|
|
||||||
/// <summary>Offset where free space starts (grows down with slots)</summary>
|
/// <summary>Offset where free space starts (grows down with slots)</summary>
|
||||||
[FieldOffset(10)]
|
[FieldOffset(10)] public ushort FreeSpaceStart;
|
||||||
public ushort FreeSpaceStart;
|
|
||||||
|
|
||||||
/// <summary>Offset where free space ends (grows up with data)</summary>
|
/// <summary>Offset where free space ends (grows up with data)</summary>
|
||||||
[FieldOffset(12)]
|
[FieldOffset(12)] public ushort FreeSpaceEnd;
|
||||||
public ushort FreeSpaceEnd;
|
|
||||||
|
|
||||||
/// <summary>Next overflow page ID (0 if none)</summary>
|
/// <summary>Next overflow page ID (0 if none)</summary>
|
||||||
[FieldOffset(14)]
|
[FieldOffset(14)] public uint NextOverflowPage;
|
||||||
public uint NextOverflowPage;
|
|
||||||
|
|
||||||
/// <summary>Transaction ID that last modified this page</summary>
|
/// <summary>Transaction ID that last modified this page</summary>
|
||||||
[FieldOffset(18)]
|
[FieldOffset(18)] public uint TransactionId;
|
||||||
public uint TransactionId;
|
|
||||||
|
|
||||||
/// <summary>Reserved for future use</summary>
|
/// <summary>Reserved for future use</summary>
|
||||||
[FieldOffset(22)]
|
[FieldOffset(22)] public ushort Reserved;
|
||||||
public ushort Reserved;
|
|
||||||
|
|
||||||
public const int Size = 24;
|
public const int Size = 24;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a header with the current slotted-page format marker.
|
/// Initializes a header with the current slotted-page format marker.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public SlottedPageHeader()
|
public SlottedPageHeader()
|
||||||
{
|
{
|
||||||
@@ -52,13 +45,13 @@ public struct SlottedPageHeader
|
|||||||
Reserved = StorageFormatConstants.SlottedPageFormatMarker;
|
Reserved = StorageFormatConstants.SlottedPageFormatMarker;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets available free space in bytes
|
/// Gets available free space in bytes
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly int AvailableFreeSpace => FreeSpaceEnd - FreeSpaceStart;
|
public readonly int AvailableFreeSpace => FreeSpaceEnd - FreeSpaceStart;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes header to span
|
/// Writes header to span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span that receives the serialized header.</param>
|
/// <param name="destination">The destination span that receives the serialized header.</param>
|
||||||
public readonly void WriteTo(Span<byte> destination)
|
public readonly void WriteTo(Span<byte> destination)
|
||||||
@@ -66,11 +59,11 @@ public struct SlottedPageHeader
|
|||||||
if (destination.Length < Size)
|
if (destination.Length < Size)
|
||||||
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
||||||
|
|
||||||
MemoryMarshal.Write(destination, in this);
|
MemoryMarshal.Write(destination, in this);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads header from span
|
/// Reads header from span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The source span containing the serialized header.</param>
|
/// <param name="source">The source span containing the serialized header.</param>
|
||||||
public static SlottedPageHeader ReadFrom(ReadOnlySpan<byte> source)
|
public static SlottedPageHeader ReadFrom(ReadOnlySpan<byte> source)
|
||||||
@@ -78,33 +71,30 @@ public struct SlottedPageHeader
|
|||||||
if (source.Length < Size)
|
if (source.Length < Size)
|
||||||
throw new ArgumentException($"Source must be at least {Size} bytes");
|
throw new ArgumentException($"Source must be at least {Size} bytes");
|
||||||
|
|
||||||
return MemoryMarshal.Read<SlottedPageHeader>(source);
|
return MemoryMarshal.Read<SlottedPageHeader>(source);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Slot entry pointing to a document within a page.
|
|
||||||
/// Fixed 8-byte structure in slot array.
|
|
||||||
/// </summary>
|
|
||||||
[StructLayout(LayoutKind.Explicit, Size = 8)]
|
|
||||||
public struct SlotEntry
|
|
||||||
{
|
|
||||||
/// <summary>Offset to document data within page</summary>
|
|
||||||
[FieldOffset(0)]
|
|
||||||
public ushort Offset;
|
|
||||||
|
|
||||||
/// <summary>Length of document data in bytes</summary>
|
/// <summary>
|
||||||
[FieldOffset(2)]
|
/// Slot entry pointing to a document within a page.
|
||||||
public ushort Length;
|
/// Fixed 8-byte structure in slot array.
|
||||||
|
/// </summary>
|
||||||
|
[StructLayout(LayoutKind.Explicit, Size = 8)]
|
||||||
|
public struct SlotEntry
|
||||||
|
{
|
||||||
|
/// <summary>Offset to document data within page</summary>
|
||||||
|
[FieldOffset(0)] public ushort Offset;
|
||||||
|
|
||||||
|
/// <summary>Length of document data in bytes</summary>
|
||||||
|
[FieldOffset(2)] public ushort Length;
|
||||||
|
|
||||||
|
/// <summary>Slot flags (deleted, overflow, etc.)</summary>
|
||||||
|
[FieldOffset(4)] public SlotFlags Flags;
|
||||||
|
|
||||||
/// <summary>Slot flags (deleted, overflow, etc.)</summary>
|
|
||||||
[FieldOffset(4)]
|
|
||||||
public SlotFlags Flags;
|
|
||||||
|
|
||||||
public const int Size = 8;
|
public const int Size = 8;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes slot entry to span
|
/// Writes slot entry to span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span that receives the serialized slot entry.</param>
|
/// <param name="destination">The destination span that receives the serialized slot entry.</param>
|
||||||
public readonly void WriteTo(Span<byte> destination)
|
public readonly void WriteTo(Span<byte> destination)
|
||||||
@@ -112,11 +102,11 @@ public struct SlotEntry
|
|||||||
if (destination.Length < Size)
|
if (destination.Length < Size)
|
||||||
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
throw new ArgumentException($"Destination must be at least {Size} bytes");
|
||||||
|
|
||||||
MemoryMarshal.Write(destination, in this);
|
MemoryMarshal.Write(destination, in this);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads slot entry from span
|
/// Reads slot entry from span
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The source span containing the serialized slot entry.</param>
|
/// <param name="source">The source span containing the serialized slot entry.</param>
|
||||||
public static SlotEntry ReadFrom(ReadOnlySpan<byte> source)
|
public static SlotEntry ReadFrom(ReadOnlySpan<byte> source)
|
||||||
@@ -124,46 +114,47 @@ public struct SlotEntry
|
|||||||
if (source.Length < Size)
|
if (source.Length < Size)
|
||||||
throw new ArgumentException($"Source must be at least {Size} bytes");
|
throw new ArgumentException($"Source must be at least {Size} bytes");
|
||||||
|
|
||||||
return MemoryMarshal.Read<SlotEntry>(source);
|
return MemoryMarshal.Read<SlotEntry>(source);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Flags for slot entries
|
/// Flags for slot entries
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Flags]
|
[Flags]
|
||||||
public enum SlotFlags : uint
|
public enum SlotFlags : uint
|
||||||
{
|
{
|
||||||
/// <summary>Slot is active and contains data</summary>
|
/// <summary>Slot is active and contains data</summary>
|
||||||
None = 0,
|
None = 0,
|
||||||
|
|
||||||
/// <summary>Slot is marked as deleted (can be reused)</summary>
|
/// <summary>Slot is marked as deleted (can be reused)</summary>
|
||||||
Deleted = 1 << 0,
|
Deleted = 1 << 0,
|
||||||
|
|
||||||
/// <summary>Document continues in overflow pages</summary>
|
/// <summary>Document continues in overflow pages</summary>
|
||||||
HasOverflow = 1 << 1,
|
HasOverflow = 1 << 1,
|
||||||
|
|
||||||
/// <summary>Document data is compressed</summary>
|
/// <summary>Document data is compressed</summary>
|
||||||
Compressed = 1 << 2,
|
Compressed = 1 << 2
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Location of a document within the database.
|
/// Location of a document within the database.
|
||||||
/// Maps ObjectId to specific page and slot.
|
/// Maps ObjectId to specific page and slot.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public readonly struct DocumentLocation
|
public readonly struct DocumentLocation
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the page identifier containing the document.
|
/// Gets the page identifier containing the document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageId { get; init; }
|
public uint PageId { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the slot index within the page.
|
/// Gets the slot index within the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ushort SlotIndex { get; init; }
|
public ushort SlotIndex { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="DocumentLocation"/> struct.
|
/// Initializes a new instance of the <see cref="DocumentLocation" /> struct.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier containing the document.</param>
|
/// <param name="pageId">The page identifier containing the document.</param>
|
||||||
/// <param name="slotIndex">The slot index within the page.</param>
|
/// <param name="slotIndex">The slot index within the page.</param>
|
||||||
@@ -174,7 +165,7 @@ public readonly struct DocumentLocation
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Serializes DocumentLocation to a byte span (6 bytes: 4 for PageId + 2 for SlotIndex)
|
/// Serializes DocumentLocation to a byte span (6 bytes: 4 for PageId + 2 for SlotIndex)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="destination">The destination span that receives the serialized value.</param>
|
/// <param name="destination">The destination span that receives the serialized value.</param>
|
||||||
public void WriteTo(Span<byte> destination)
|
public void WriteTo(Span<byte> destination)
|
||||||
@@ -182,12 +173,12 @@ public readonly struct DocumentLocation
|
|||||||
if (destination.Length < 6)
|
if (destination.Length < 6)
|
||||||
throw new ArgumentException("Destination must be at least 6 bytes", nameof(destination));
|
throw new ArgumentException("Destination must be at least 6 bytes", nameof(destination));
|
||||||
|
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(destination, PageId);
|
BinaryPrimitives.WriteUInt32LittleEndian(destination, PageId);
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteUInt16LittleEndian(destination.Slice(4), SlotIndex);
|
BinaryPrimitives.WriteUInt16LittleEndian(destination.Slice(4), SlotIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Deserializes DocumentLocation from a byte span (6 bytes)
|
/// Deserializes DocumentLocation from a byte span (6 bytes)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="source">The source span containing the serialized value.</param>
|
/// <param name="source">The source span containing the serialized value.</param>
|
||||||
public static DocumentLocation ReadFrom(ReadOnlySpan<byte> source)
|
public static DocumentLocation ReadFrom(ReadOnlySpan<byte> source)
|
||||||
@@ -195,14 +186,14 @@ public readonly struct DocumentLocation
|
|||||||
if (source.Length < 6)
|
if (source.Length < 6)
|
||||||
throw new ArgumentException("Source must be at least 6 bytes", nameof(source));
|
throw new ArgumentException("Source must be at least 6 bytes", nameof(source));
|
||||||
|
|
||||||
var pageId = System.Buffers.Binary.BinaryPrimitives.ReadUInt32LittleEndian(source);
|
uint pageId = BinaryPrimitives.ReadUInt32LittleEndian(source);
|
||||||
var slotIndex = System.Buffers.Binary.BinaryPrimitives.ReadUInt16LittleEndian(source.Slice(4));
|
ushort slotIndex = BinaryPrimitives.ReadUInt16LittleEndian(source.Slice(4));
|
||||||
|
|
||||||
return new DocumentLocation(pageId, slotIndex);
|
return new DocumentLocation(pageId, slotIndex);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Size in bytes when serialized
|
/// Size in bytes when serialized
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public const int SerializedSize = 6;
|
public const int SerializedSize = 6;
|
||||||
}
|
}
|
||||||
@@ -1,12 +1,11 @@
|
|||||||
using System.Buffers.Binary;
|
using System.Buffers.Binary;
|
||||||
using System.Runtime.InteropServices;
|
using System.Runtime.InteropServices;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
using ZB.MOM.WW.CBDD.Core.Indexing.Internal;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Page for storing R-Tree nodes for Geospatial Indexing.
|
/// Page for storing R-Tree nodes for Geospatial Indexing.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal struct SpatialPage
|
internal struct SpatialPage
|
||||||
{
|
{
|
||||||
@@ -29,14 +28,14 @@ internal struct SpatialPage
|
|||||||
|
|
||||||
public const int EntrySize = 38; // 32 (GeoBox) + 6 (Pointer)
|
public const int EntrySize = 38; // 32 (GeoBox) + 6 (Pointer)
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a spatial page.
|
/// Initializes a spatial page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer to initialize.</param>
|
/// <param name="page">The page buffer to initialize.</param>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
/// <param name="isLeaf">Whether this page is a leaf node.</param>
|
/// <param name="isLeaf">Whether this page is a leaf node.</param>
|
||||||
/// <param name="level">The tree level for this page.</param>
|
/// <param name="level">The tree level for this page.</param>
|
||||||
public static void Initialize(Span<byte> page, uint pageId, bool isLeaf, byte level)
|
public static void Initialize(Span<byte> page, uint pageId, bool isLeaf, byte level)
|
||||||
{
|
{
|
||||||
var header = new PageHeader
|
var header = new PageHeader
|
||||||
{
|
{
|
||||||
@@ -54,65 +53,86 @@ internal struct SpatialPage
|
|||||||
BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), 0);
|
BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether the page is a leaf node.
|
/// Gets a value indicating whether the page is a leaf node.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns><see langword="true"/> if the page is a leaf node; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if the page is a leaf node; otherwise, <see langword="false" />.</returns>
|
||||||
public static bool GetIsLeaf(ReadOnlySpan<byte> page) => page[IsLeafOffset] == 1;
|
public static bool GetIsLeaf(ReadOnlySpan<byte> page)
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the tree level stored in the page.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <returns>The level value.</returns>
|
|
||||||
public static byte GetLevel(ReadOnlySpan<byte> page) => page[LevelOffset];
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the number of entries in the page.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <returns>The number of entries.</returns>
|
|
||||||
public static ushort GetEntryCount(ReadOnlySpan<byte> page) => BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(EntryCountOffset));
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Sets the number of entries in the page.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <param name="count">The entry count to set.</param>
|
|
||||||
public static void SetEntryCount(Span<byte> page, ushort count) => BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(EntryCountOffset), count);
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the parent page identifier.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <returns>The parent page identifier.</returns>
|
|
||||||
public static uint GetParentPageId(ReadOnlySpan<byte> page) => BinaryPrimitives.ReadUInt32LittleEndian(page.Slice(ParentPageIdOffset));
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Sets the parent page identifier.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <param name="parentId">The parent page identifier.</param>
|
|
||||||
public static void SetParentPageId(Span<byte> page, uint parentId) => BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), parentId);
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the maximum number of entries that can fit in a page.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="pageSize">The page size in bytes.</param>
|
|
||||||
/// <returns>The maximum number of entries.</returns>
|
|
||||||
public static int GetMaxEntries(int pageSize) => (pageSize - DataOffset) / EntrySize;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Writes an entry at the specified index.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="page">The page buffer.</param>
|
|
||||||
/// <param name="index">The entry index.</param>
|
|
||||||
/// <param name="mbr">The minimum bounding rectangle for the entry.</param>
|
|
||||||
/// <param name="pointer">The document location pointer.</param>
|
|
||||||
public static void WriteEntry(Span<byte> page, int index, GeoBox mbr, DocumentLocation pointer)
|
|
||||||
{
|
{
|
||||||
int offset = DataOffset + (index * EntrySize);
|
return page[IsLeafOffset] == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the tree level stored in the page.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <returns>The level value.</returns>
|
||||||
|
public static byte GetLevel(ReadOnlySpan<byte> page)
|
||||||
|
{
|
||||||
|
return page[LevelOffset];
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the number of entries in the page.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <returns>The number of entries.</returns>
|
||||||
|
public static ushort GetEntryCount(ReadOnlySpan<byte> page)
|
||||||
|
{
|
||||||
|
return BinaryPrimitives.ReadUInt16LittleEndian(page.Slice(EntryCountOffset));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Sets the number of entries in the page.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <param name="count">The entry count to set.</param>
|
||||||
|
public static void SetEntryCount(Span<byte> page, ushort count)
|
||||||
|
{
|
||||||
|
BinaryPrimitives.WriteUInt16LittleEndian(page.Slice(EntryCountOffset), count);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the parent page identifier.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <returns>The parent page identifier.</returns>
|
||||||
|
public static uint GetParentPageId(ReadOnlySpan<byte> page)
|
||||||
|
{
|
||||||
|
return BinaryPrimitives.ReadUInt32LittleEndian(page.Slice(ParentPageIdOffset));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Sets the parent page identifier.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <param name="parentId">The parent page identifier.</param>
|
||||||
|
public static void SetParentPageId(Span<byte> page, uint parentId)
|
||||||
|
{
|
||||||
|
BinaryPrimitives.WriteUInt32LittleEndian(page.Slice(ParentPageIdOffset), parentId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the maximum number of entries that can fit in a page.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="pageSize">The page size in bytes.</param>
|
||||||
|
/// <returns>The maximum number of entries.</returns>
|
||||||
|
public static int GetMaxEntries(int pageSize)
|
||||||
|
{
|
||||||
|
return (pageSize - DataOffset) / EntrySize;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Writes an entry at the specified index.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="page">The page buffer.</param>
|
||||||
|
/// <param name="index">The entry index.</param>
|
||||||
|
/// <param name="mbr">The minimum bounding rectangle for the entry.</param>
|
||||||
|
/// <param name="pointer">The document location pointer.</param>
|
||||||
|
public static void WriteEntry(Span<byte> page, int index, GeoBox mbr, DocumentLocation pointer)
|
||||||
|
{
|
||||||
|
int offset = DataOffset + index * EntrySize;
|
||||||
var entrySpan = page.Slice(offset, EntrySize);
|
var entrySpan = page.Slice(offset, EntrySize);
|
||||||
|
|
||||||
// Write MBR (4 doubles)
|
// Write MBR (4 doubles)
|
||||||
@@ -126,16 +146,16 @@ internal struct SpatialPage
|
|||||||
pointer.WriteTo(entrySpan.Slice(32, 6));
|
pointer.WriteTo(entrySpan.Slice(32, 6));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads an entry at the specified index.
|
/// Reads an entry at the specified index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="index">The entry index.</param>
|
/// <param name="index">The entry index.</param>
|
||||||
/// <param name="mbr">When this method returns, contains the entry MBR.</param>
|
/// <param name="mbr">When this method returns, contains the entry MBR.</param>
|
||||||
/// <param name="pointer">When this method returns, contains the entry document location.</param>
|
/// <param name="pointer">When this method returns, contains the entry document location.</param>
|
||||||
public static void ReadEntry(ReadOnlySpan<byte> page, int index, out GeoBox mbr, out DocumentLocation pointer)
|
public static void ReadEntry(ReadOnlySpan<byte> page, int index, out GeoBox mbr, out DocumentLocation pointer)
|
||||||
{
|
{
|
||||||
int offset = DataOffset + (index * EntrySize);
|
int offset = DataOffset + index * EntrySize;
|
||||||
var entrySpan = page.Slice(offset, EntrySize);
|
var entrySpan = page.Slice(offset, EntrySize);
|
||||||
|
|
||||||
var doubles = MemoryMarshal.Cast<byte, double>(entrySpan.Slice(0, 32));
|
var doubles = MemoryMarshal.Cast<byte, double>(entrySpan.Slice(0, 32));
|
||||||
@@ -143,23 +163,24 @@ internal struct SpatialPage
|
|||||||
pointer = DocumentLocation.ReadFrom(entrySpan.Slice(32, 6));
|
pointer = DocumentLocation.ReadFrom(entrySpan.Slice(32, 6));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Calculates the combined MBR of all entries in the page.
|
/// Calculates the combined MBR of all entries in the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns>The combined MBR, or <see cref="GeoBox.Empty"/> when the page has no entries.</returns>
|
/// <returns>The combined MBR, or <see cref="GeoBox.Empty" /> when the page has no entries.</returns>
|
||||||
public static GeoBox CalculateMBR(ReadOnlySpan<byte> page)
|
public static GeoBox CalculateMBR(ReadOnlySpan<byte> page)
|
||||||
{
|
{
|
||||||
ushort count = GetEntryCount(page);
|
ushort count = GetEntryCount(page);
|
||||||
if (count == 0) return GeoBox.Empty;
|
if (count == 0) return GeoBox.Empty;
|
||||||
|
|
||||||
GeoBox result = GeoBox.Empty;
|
var result = GeoBox.Empty;
|
||||||
for (int i = 0; i < count; i++)
|
for (var i = 0; i < count; i++)
|
||||||
{
|
{
|
||||||
ReadEntry(page, i, out var mbr, out _);
|
ReadEntry(page, i, out var mbr, out _);
|
||||||
if (i == 0) result = mbr;
|
if (i == 0) result = mbr;
|
||||||
else result = result.ExpandTo(mbr);
|
else result = result.ExpandTo(mbr);
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,31 +1,27 @@
|
|||||||
using System;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using System.Collections.Generic;
|
using ZB.MOM.WW.CBDD.Core.Indexing;
|
||||||
using System.IO;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
|
||||||
|
|
||||||
public class CollectionMetadata
|
public class CollectionMetadata
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the collection name.
|
/// Gets or sets the collection name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string Name { get; set; } = string.Empty;
|
public string Name { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the root page identifier of the primary index.
|
/// Gets or sets the root page identifier of the primary index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PrimaryRootPageId { get; set; }
|
public uint PrimaryRootPageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the root page identifier of the schema chain.
|
/// Gets or sets the root page identifier of the schema chain.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint SchemaRootPageId { get; set; }
|
public uint SchemaRootPageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection index metadata list.
|
/// Gets the collection index metadata list.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<IndexMetadata> Indexes { get; } = new();
|
public List<IndexMetadata> Indexes { get; } = new();
|
||||||
}
|
}
|
||||||
@@ -33,45 +29,45 @@ public class CollectionMetadata
|
|||||||
public class IndexMetadata
|
public class IndexMetadata
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the index name.
|
/// Gets or sets the index name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string Name { get; set; } = string.Empty;
|
public string Name { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets a value indicating whether this index enforces uniqueness.
|
/// Gets or sets a value indicating whether this index enforces uniqueness.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsUnique { get; set; }
|
public bool IsUnique { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the index type.
|
/// Gets or sets the index type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IndexType Type { get; set; }
|
public IndexType Type { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets indexed property paths.
|
/// Gets or sets indexed property paths.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string[] PropertyPaths { get; set; } = Array.Empty<string>();
|
public string[] PropertyPaths { get; set; } = Array.Empty<string>();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets vector dimensions for vector indexes.
|
/// Gets or sets vector dimensions for vector indexes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Dimensions { get; set; }
|
public int Dimensions { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the vector similarity metric for vector indexes.
|
/// Gets or sets the vector similarity metric for vector indexes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public VectorMetric Metric { get; set; }
|
public VectorMetric Metric { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the root page identifier of the index structure.
|
/// Gets or sets the root page identifier of the index structure.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint RootPageId { get; set; }
|
public uint RootPageId { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets collection metadata by name.
|
/// Gets collection metadata by name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="name">The collection name.</param>
|
/// <param name="name">The collection name.</param>
|
||||||
/// <returns>The collection metadata if found; otherwise, null.</returns>
|
/// <returns>The collection metadata if found; otherwise, null.</returns>
|
||||||
@@ -82,7 +78,121 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns all collection metadata entries currently registered in page 1.
|
/// Saves collection metadata to the metadata page.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="metadata">The metadata to save.</param>
|
||||||
|
public void SaveCollectionMetadata(CollectionMetadata metadata)
|
||||||
|
{
|
||||||
|
using var stream = new MemoryStream();
|
||||||
|
using var writer = new BinaryWriter(stream);
|
||||||
|
|
||||||
|
writer.Write(metadata.Name);
|
||||||
|
writer.Write(metadata.PrimaryRootPageId);
|
||||||
|
writer.Write(metadata.SchemaRootPageId);
|
||||||
|
writer.Write(metadata.Indexes.Count);
|
||||||
|
foreach (var idx in metadata.Indexes)
|
||||||
|
{
|
||||||
|
writer.Write(idx.Name);
|
||||||
|
writer.Write(idx.IsUnique);
|
||||||
|
writer.Write((byte)idx.Type);
|
||||||
|
writer.Write(idx.RootPageId);
|
||||||
|
writer.Write(idx.PropertyPaths.Length);
|
||||||
|
foreach (string path in idx.PropertyPaths) writer.Write(path);
|
||||||
|
|
||||||
|
if (idx.Type == IndexType.Vector)
|
||||||
|
{
|
||||||
|
writer.Write(idx.Dimensions);
|
||||||
|
writer.Write((byte)idx.Metric);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
byte[] newData = stream.ToArray();
|
||||||
|
|
||||||
|
var buffer = new byte[PageSize];
|
||||||
|
ReadPage(1, null, buffer);
|
||||||
|
|
||||||
|
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||||
|
int existingSlotIndex = -1;
|
||||||
|
|
||||||
|
for (ushort i = 0; i < header.SlotCount; i++)
|
||||||
|
{
|
||||||
|
int slotOffset = SlottedPageHeader.Size + i * SlotEntry.Size;
|
||||||
|
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||||
|
if ((slot.Flags & SlotFlags.Deleted) != 0) continue;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
using var ms = new MemoryStream(buffer, slot.Offset, slot.Length, false);
|
||||||
|
using var reader = new BinaryReader(ms);
|
||||||
|
string name = reader.ReadString();
|
||||||
|
|
||||||
|
if (string.Equals(name, metadata.Name, StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
existingSlotIndex = i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existingSlotIndex >= 0)
|
||||||
|
{
|
||||||
|
int slotOffset = SlottedPageHeader.Size + existingSlotIndex * SlotEntry.Size;
|
||||||
|
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||||
|
slot.Flags |= SlotFlags.Deleted;
|
||||||
|
slot.WriteTo(buffer.AsSpan(slotOffset));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (header.AvailableFreeSpace < newData.Length + SlotEntry.Size)
|
||||||
|
// Compact logic omitted as per current architecture
|
||||||
|
throw new InvalidOperationException(
|
||||||
|
"Not enough space in Metadata Page (Page 1) to save collection metadata.");
|
||||||
|
|
||||||
|
int docOffset = header.FreeSpaceEnd - newData.Length;
|
||||||
|
newData.CopyTo(buffer.AsSpan(docOffset));
|
||||||
|
|
||||||
|
ushort slotIndex;
|
||||||
|
if (existingSlotIndex >= 0)
|
||||||
|
{
|
||||||
|
slotIndex = (ushort)existingSlotIndex;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
slotIndex = header.SlotCount;
|
||||||
|
header.SlotCount++;
|
||||||
|
}
|
||||||
|
|
||||||
|
int newSlotEntryOffset = SlottedPageHeader.Size + slotIndex * SlotEntry.Size;
|
||||||
|
var newSlot = new SlotEntry
|
||||||
|
{
|
||||||
|
Offset = (ushort)docOffset,
|
||||||
|
Length = (ushort)newData.Length,
|
||||||
|
Flags = SlotFlags.None
|
||||||
|
};
|
||||||
|
newSlot.WriteTo(buffer.AsSpan(newSlotEntryOffset));
|
||||||
|
|
||||||
|
header.FreeSpaceEnd = (ushort)docOffset;
|
||||||
|
if (existingSlotIndex == -1)
|
||||||
|
header.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + header.SlotCount * SlotEntry.Size);
|
||||||
|
|
||||||
|
header.WriteTo(buffer);
|
||||||
|
WritePageImmediate(1, buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Registers all BSON keys used by a set of mappers into the global dictionary.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="mappers">The mappers whose keys should be registered.</param>
|
||||||
|
public void RegisterMappers(IEnumerable<IDocumentMapper> mappers)
|
||||||
|
{
|
||||||
|
var allKeys = mappers.SelectMany(m => m.UsedKeys).Distinct();
|
||||||
|
RegisterKeys(allKeys);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns all collection metadata entries currently registered in page 1.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<CollectionMetadata> GetAllCollectionMetadata()
|
public IReadOnlyList<CollectionMetadata> GetAllCollectionMetadata()
|
||||||
{
|
{
|
||||||
@@ -96,7 +206,7 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
for (ushort i = 0; i < header.SlotCount; i++)
|
for (ushort i = 0; i < header.SlotCount; i++)
|
||||||
{
|
{
|
||||||
var slotOffset = SlottedPageHeader.Size + (i * SlotEntry.Size);
|
int slotOffset = SlottedPageHeader.Size + i * SlotEntry.Size;
|
||||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||||
continue;
|
continue;
|
||||||
@@ -104,122 +214,12 @@ public sealed partial class StorageEngine
|
|||||||
if (slot.Offset < SlottedPageHeader.Size || slot.Offset + slot.Length > buffer.Length)
|
if (slot.Offset < SlottedPageHeader.Size || slot.Offset + slot.Length > buffer.Length)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
if (TryDeserializeCollectionMetadata(buffer.AsSpan(slot.Offset, slot.Length), out var metadata) && metadata != null)
|
if (TryDeserializeCollectionMetadata(buffer.AsSpan(slot.Offset, slot.Length), out var metadata) &&
|
||||||
{
|
metadata != null) result.Add(metadata);
|
||||||
result.Add(metadata);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Saves collection metadata to the metadata page.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="metadata">The metadata to save.</param>
|
|
||||||
public void SaveCollectionMetadata(CollectionMetadata metadata)
|
|
||||||
{
|
|
||||||
using var stream = new MemoryStream();
|
|
||||||
using var writer = new BinaryWriter(stream);
|
|
||||||
|
|
||||||
writer.Write(metadata.Name);
|
|
||||||
writer.Write(metadata.PrimaryRootPageId);
|
|
||||||
writer.Write(metadata.SchemaRootPageId);
|
|
||||||
writer.Write(metadata.Indexes.Count);
|
|
||||||
foreach (var idx in metadata.Indexes)
|
|
||||||
{
|
|
||||||
writer.Write(idx.Name);
|
|
||||||
writer.Write(idx.IsUnique);
|
|
||||||
writer.Write((byte)idx.Type);
|
|
||||||
writer.Write(idx.RootPageId);
|
|
||||||
writer.Write(idx.PropertyPaths.Length);
|
|
||||||
foreach (var path in idx.PropertyPaths)
|
|
||||||
{
|
|
||||||
writer.Write(path);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (idx.Type == IndexType.Vector)
|
|
||||||
{
|
|
||||||
writer.Write(idx.Dimensions);
|
|
||||||
writer.Write((byte)idx.Metric);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var newData = stream.ToArray();
|
|
||||||
|
|
||||||
var buffer = new byte[PageSize];
|
|
||||||
ReadPage(1, null, buffer);
|
|
||||||
|
|
||||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
|
||||||
int existingSlotIndex = -1;
|
|
||||||
|
|
||||||
for (ushort i = 0; i < header.SlotCount; i++)
|
|
||||||
{
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (i * SlotEntry.Size);
|
|
||||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0) continue;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
using var ms = new MemoryStream(buffer, slot.Offset, slot.Length, false);
|
|
||||||
using var reader = new BinaryReader(ms);
|
|
||||||
var name = reader.ReadString();
|
|
||||||
|
|
||||||
if (string.Equals(name, metadata.Name, StringComparison.OrdinalIgnoreCase))
|
|
||||||
{
|
|
||||||
existingSlotIndex = i;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
|
|
||||||
if (existingSlotIndex >= 0)
|
|
||||||
{
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (existingSlotIndex * SlotEntry.Size);
|
|
||||||
var slot = SlotEntry.ReadFrom(buffer.AsSpan(slotOffset));
|
|
||||||
slot.Flags |= SlotFlags.Deleted;
|
|
||||||
slot.WriteTo(buffer.AsSpan(slotOffset));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (header.AvailableFreeSpace < newData.Length + SlotEntry.Size)
|
|
||||||
{
|
|
||||||
// Compact logic omitted as per current architecture
|
|
||||||
throw new InvalidOperationException("Not enough space in Metadata Page (Page 1) to save collection metadata.");
|
|
||||||
}
|
|
||||||
|
|
||||||
int docOffset = header.FreeSpaceEnd - newData.Length;
|
|
||||||
newData.CopyTo(buffer.AsSpan(docOffset));
|
|
||||||
|
|
||||||
ushort slotIndex;
|
|
||||||
if (existingSlotIndex >= 0)
|
|
||||||
{
|
|
||||||
slotIndex = (ushort)existingSlotIndex;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
slotIndex = header.SlotCount;
|
|
||||||
header.SlotCount++;
|
|
||||||
}
|
|
||||||
|
|
||||||
var newSlotEntryOffset = SlottedPageHeader.Size + (slotIndex * SlotEntry.Size);
|
|
||||||
var newSlot = new SlotEntry
|
|
||||||
{
|
|
||||||
Offset = (ushort)docOffset,
|
|
||||||
Length = (ushort)newData.Length,
|
|
||||||
Flags = SlotFlags.None
|
|
||||||
};
|
|
||||||
newSlot.WriteTo(buffer.AsSpan(newSlotEntryOffset));
|
|
||||||
|
|
||||||
header.FreeSpaceEnd = (ushort)docOffset;
|
|
||||||
if (existingSlotIndex == -1)
|
|
||||||
{
|
|
||||||
header.FreeSpaceStart = (ushort)(SlottedPageHeader.Size + (header.SlotCount * SlotEntry.Size));
|
|
||||||
}
|
|
||||||
|
|
||||||
header.WriteTo(buffer);
|
|
||||||
WritePageImmediate(1, buffer);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool TryDeserializeCollectionMetadata(ReadOnlySpan<byte> rawBytes, out CollectionMetadata? metadata)
|
private static bool TryDeserializeCollectionMetadata(ReadOnlySpan<byte> rawBytes, out CollectionMetadata? metadata)
|
||||||
{
|
{
|
||||||
@@ -230,16 +230,16 @@ public sealed partial class StorageEngine
|
|||||||
using var ms = new MemoryStream(rawBytes.ToArray());
|
using var ms = new MemoryStream(rawBytes.ToArray());
|
||||||
using var reader = new BinaryReader(ms);
|
using var reader = new BinaryReader(ms);
|
||||||
|
|
||||||
var collName = reader.ReadString();
|
string collName = reader.ReadString();
|
||||||
var parsed = new CollectionMetadata { Name = collName };
|
var parsed = new CollectionMetadata { Name = collName };
|
||||||
parsed.PrimaryRootPageId = reader.ReadUInt32();
|
parsed.PrimaryRootPageId = reader.ReadUInt32();
|
||||||
parsed.SchemaRootPageId = reader.ReadUInt32();
|
parsed.SchemaRootPageId = reader.ReadUInt32();
|
||||||
|
|
||||||
var indexCount = reader.ReadInt32();
|
int indexCount = reader.ReadInt32();
|
||||||
if (indexCount < 0)
|
if (indexCount < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
for (int j = 0; j < indexCount; j++)
|
for (var j = 0; j < indexCount; j++)
|
||||||
{
|
{
|
||||||
var idx = new IndexMetadata
|
var idx = new IndexMetadata
|
||||||
{
|
{
|
||||||
@@ -249,12 +249,12 @@ public sealed partial class StorageEngine
|
|||||||
RootPageId = reader.ReadUInt32()
|
RootPageId = reader.ReadUInt32()
|
||||||
};
|
};
|
||||||
|
|
||||||
var pathCount = reader.ReadInt32();
|
int pathCount = reader.ReadInt32();
|
||||||
if (pathCount < 0)
|
if (pathCount < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
idx.PropertyPaths = new string[pathCount];
|
idx.PropertyPaths = new string[pathCount];
|
||||||
for (int k = 0; k < pathCount; k++)
|
for (var k = 0; k < pathCount; k++)
|
||||||
idx.PropertyPaths[k] = reader.ReadString();
|
idx.PropertyPaths[k] = reader.ReadString();
|
||||||
|
|
||||||
if (idx.Type == IndexType.Vector)
|
if (idx.Type == IndexType.Vector)
|
||||||
@@ -274,14 +274,4 @@ public sealed partial class StorageEngine
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Registers all BSON keys used by a set of mappers into the global dictionary.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="mappers">The mappers whose keys should be registered.</param>
|
|
||||||
public void RegisterMappers(IEnumerable<IDocumentMapper> mappers)
|
|
||||||
{
|
|
||||||
var allKeys = mappers.SelectMany(m => m.UsedKeys).Distinct();
|
|
||||||
RegisterKeys(allKeys);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,172 +5,173 @@ using ZB.MOM.WW.CBDD.Core.Indexing;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Aggregated page counts grouped by page type.
|
/// Aggregated page counts grouped by page type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class PageTypeUsageEntry
|
public sealed class PageTypeUsageEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the page type.
|
/// Gets the page type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public PageType PageType { get; init; }
|
public PageType PageType { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of pages of this type.
|
/// Gets the number of pages of this type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int PageCount { get; init; }
|
public int PageCount { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Per-collection page usage summary.
|
/// Per-collection page usage summary.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CollectionPageUsageEntry
|
public sealed class CollectionPageUsageEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection name.
|
/// Gets the collection name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string CollectionName { get; init; } = string.Empty;
|
public string CollectionName { get; init; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total number of distinct pages referenced by the collection.
|
/// Gets the total number of distinct pages referenced by the collection.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int TotalDistinctPages { get; init; }
|
public int TotalDistinctPages { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of data pages.
|
/// Gets the number of data pages.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int DataPages { get; init; }
|
public int DataPages { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of overflow pages.
|
/// Gets the number of overflow pages.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int OverflowPages { get; init; }
|
public int OverflowPages { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of index pages.
|
/// Gets the number of index pages.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int IndexPages { get; init; }
|
public int IndexPages { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of other page types.
|
/// Gets the number of other page types.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int OtherPages { get; init; }
|
public int OtherPages { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Per-collection compression ratio summary.
|
/// Per-collection compression ratio summary.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CollectionCompressionRatioEntry
|
public sealed class CollectionCompressionRatioEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the collection name.
|
/// Gets the collection name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string CollectionName { get; init; } = string.Empty;
|
public string CollectionName { get; init; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of documents.
|
/// Gets the number of documents.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DocumentCount { get; init; }
|
public long DocumentCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of compressed documents.
|
/// Gets the number of compressed documents.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long CompressedDocumentCount { get; init; }
|
public long CompressedDocumentCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total uncompressed byte count.
|
/// Gets the total uncompressed byte count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesBeforeCompression { get; init; }
|
public long BytesBeforeCompression { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total stored byte count.
|
/// Gets the total stored byte count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesAfterCompression { get; init; }
|
public long BytesAfterCompression { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the compression ratio.
|
/// Gets the compression ratio.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public double CompressionRatio => BytesAfterCompression <= 0 ? 1.0 : (double)BytesBeforeCompression / BytesAfterCompression;
|
public double CompressionRatio =>
|
||||||
|
BytesAfterCompression <= 0 ? 1.0 : (double)BytesBeforeCompression / BytesAfterCompression;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Summary of free-list and reclaimable tail information.
|
/// Summary of free-list and reclaimable tail information.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class FreeListSummary
|
public sealed class FreeListSummary
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total page count.
|
/// Gets the total page count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageCount { get; init; }
|
public uint PageCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the free page count.
|
/// Gets the free page count.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int FreePageCount { get; init; }
|
public int FreePageCount { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total free bytes.
|
/// Gets the total free bytes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long FreeBytes { get; init; }
|
public long FreeBytes { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the fragmentation percentage.
|
/// Gets the fragmentation percentage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public double FragmentationPercent { get; init; }
|
public double FragmentationPercent { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of reclaimable pages at the file tail.
|
/// Gets the number of reclaimable pages at the file tail.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint TailReclaimablePages { get; init; }
|
public uint TailReclaimablePages { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Single page entry in fragmentation reporting.
|
/// Single page entry in fragmentation reporting.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class FragmentationPageEntry
|
public sealed class FragmentationPageEntry
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the page identifier.
|
/// Gets the page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageId { get; init; }
|
public uint PageId { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the page type.
|
/// Gets the page type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public PageType PageType { get; init; }
|
public PageType PageType { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether this page is free.
|
/// Gets a value indicating whether this page is free.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsFreePage { get; init; }
|
public bool IsFreePage { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the free bytes on the page.
|
/// Gets the free bytes on the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int FreeBytes { get; init; }
|
public int FreeBytes { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Detailed fragmentation map and totals.
|
/// Detailed fragmentation map and totals.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class FragmentationMapReport
|
public sealed class FragmentationMapReport
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the page entries.
|
/// Gets the page entries.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<FragmentationPageEntry> Pages { get; init; } = Array.Empty<FragmentationPageEntry>();
|
public IReadOnlyList<FragmentationPageEntry> Pages { get; init; } = Array.Empty<FragmentationPageEntry>();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total free bytes across all pages.
|
/// Gets the total free bytes across all pages.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long TotalFreeBytes { get; init; }
|
public long TotalFreeBytes { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the fragmentation percentage.
|
/// Gets the fragmentation percentage.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public double FragmentationPercent { get; init; }
|
public double FragmentationPercent { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of reclaimable pages at the file tail.
|
/// Gets the number of reclaimable pages at the file tail.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint TailReclaimablePages { get; init; }
|
public uint TailReclaimablePages { get; init; }
|
||||||
}
|
}
|
||||||
@@ -178,11 +179,11 @@ public sealed class FragmentationMapReport
|
|||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets page usage grouped by page type.
|
/// Gets page usage grouped by page type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
public IReadOnlyList<PageTypeUsageEntry> GetPageUsageByPageType()
|
||||||
{
|
{
|
||||||
var pageCount = _pageFile.NextPageId;
|
uint pageCount = _pageFile.NextPageId;
|
||||||
var buffer = new byte[_pageFile.PageSize];
|
var buffer = new byte[_pageFile.PageSize];
|
||||||
var counts = new Dictionary<PageType, int>();
|
var counts = new Dictionary<PageType, int>();
|
||||||
|
|
||||||
@@ -190,7 +191,7 @@ public sealed partial class StorageEngine
|
|||||||
{
|
{
|
||||||
_pageFile.ReadPage(pageId, buffer);
|
_pageFile.ReadPage(pageId, buffer);
|
||||||
var pageType = PageHeader.ReadFrom(buffer).PageType;
|
var pageType = PageHeader.ReadFrom(buffer).PageType;
|
||||||
counts[pageType] = counts.TryGetValue(pageType, out var count) ? count + 1 : 1;
|
counts[pageType] = counts.TryGetValue(pageType, out int count) ? count + 1 : 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return counts
|
return counts
|
||||||
@@ -204,7 +205,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets per-collection page usage by resolving primary-index locations and related index roots.
|
/// Gets per-collection page usage by resolving primary-index locations and related index roots.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
public IReadOnlyList<CollectionPageUsageEntry> GetPageUsageByCollection()
|
||||||
{
|
{
|
||||||
@@ -221,27 +222,23 @@ public sealed partial class StorageEngine
|
|||||||
pageIds.Add(metadata.SchemaRootPageId);
|
pageIds.Add(metadata.SchemaRootPageId);
|
||||||
|
|
||||||
foreach (var indexMetadata in metadata.Indexes)
|
foreach (var indexMetadata in metadata.Indexes)
|
||||||
{
|
|
||||||
if (indexMetadata.RootPageId != 0)
|
if (indexMetadata.RootPageId != 0)
|
||||||
pageIds.Add(indexMetadata.RootPageId);
|
pageIds.Add(indexMetadata.RootPageId);
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||||
{
|
{
|
||||||
pageIds.Add(location.PageId);
|
pageIds.Add(location.PageId);
|
||||||
if (TryReadFirstOverflowPage(location, out var firstOverflowPage))
|
if (TryReadFirstOverflowPage(location, out uint firstOverflowPage))
|
||||||
{
|
|
||||||
AddOverflowChainPages(pageIds, firstOverflowPage);
|
AddOverflowChainPages(pageIds, firstOverflowPage);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int data = 0;
|
var data = 0;
|
||||||
int overflow = 0;
|
var overflow = 0;
|
||||||
int indexPages = 0;
|
var indexPages = 0;
|
||||||
int other = 0;
|
var other = 0;
|
||||||
|
|
||||||
var pageBuffer = new byte[_pageFile.PageSize];
|
var pageBuffer = new byte[_pageFile.PageSize];
|
||||||
foreach (var pageId in pageIds)
|
foreach (uint pageId in pageIds)
|
||||||
{
|
{
|
||||||
if (pageId >= _pageFile.NextPageId)
|
if (pageId >= _pageFile.NextPageId)
|
||||||
continue;
|
continue;
|
||||||
@@ -250,21 +247,13 @@ public sealed partial class StorageEngine
|
|||||||
var pageType = PageHeader.ReadFrom(pageBuffer).PageType;
|
var pageType = PageHeader.ReadFrom(pageBuffer).PageType;
|
||||||
|
|
||||||
if (pageType == PageType.Data)
|
if (pageType == PageType.Data)
|
||||||
{
|
|
||||||
data++;
|
data++;
|
||||||
}
|
|
||||||
else if (pageType == PageType.Overflow)
|
else if (pageType == PageType.Overflow)
|
||||||
{
|
|
||||||
overflow++;
|
overflow++;
|
||||||
}
|
|
||||||
else if (pageType == PageType.Index || pageType == PageType.Vector || pageType == PageType.Spatial)
|
else if (pageType == PageType.Index || pageType == PageType.Vector || pageType == PageType.Spatial)
|
||||||
{
|
|
||||||
indexPages++;
|
indexPages++;
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
|
||||||
other++;
|
other++;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
results.Add(new CollectionPageUsageEntry
|
results.Add(new CollectionPageUsageEntry
|
||||||
@@ -282,7 +271,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets per-collection logical-vs-stored compression ratios.
|
/// Gets per-collection logical-vs-stored compression ratios.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
public IReadOnlyList<CollectionCompressionRatioEntry> GetCompressionRatioByCollection()
|
||||||
{
|
{
|
||||||
@@ -298,7 +287,8 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
foreach (var location in EnumeratePrimaryLocations(metadata))
|
foreach (var location in EnumeratePrimaryLocations(metadata))
|
||||||
{
|
{
|
||||||
if (!TryReadSlotPayloadStats(location, out var isCompressed, out var originalBytes, out var storedBytes))
|
if (!TryReadSlotPayloadStats(location, out bool isCompressed, out int originalBytes,
|
||||||
|
out int storedBytes))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
docs++;
|
docs++;
|
||||||
@@ -323,7 +313,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets free-list summary for diagnostics.
|
/// Gets free-list summary for diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public FreeListSummary GetFreeListSummary()
|
public FreeListSummary GetFreeListSummary()
|
||||||
{
|
{
|
||||||
@@ -339,12 +329,12 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets detailed page-level fragmentation diagnostics.
|
/// Gets detailed page-level fragmentation diagnostics.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public FragmentationMapReport GetFragmentationMap()
|
public FragmentationMapReport GetFragmentationMap()
|
||||||
{
|
{
|
||||||
var freePageSet = new HashSet<uint>(_pageFile.EnumerateFreePages(includeEmptyPages: true));
|
var freePageSet = new HashSet<uint>(_pageFile.EnumerateFreePages());
|
||||||
var pageCount = _pageFile.NextPageId;
|
uint pageCount = _pageFile.NextPageId;
|
||||||
var buffer = new byte[_pageFile.PageSize];
|
var buffer = new byte[_pageFile.PageSize];
|
||||||
var pages = new List<FragmentationPageEntry>((int)pageCount);
|
var pages = new List<FragmentationPageEntry>((int)pageCount);
|
||||||
|
|
||||||
@@ -354,17 +344,12 @@ public sealed partial class StorageEngine
|
|||||||
{
|
{
|
||||||
_pageFile.ReadPage(pageId, buffer);
|
_pageFile.ReadPage(pageId, buffer);
|
||||||
var pageHeader = PageHeader.ReadFrom(buffer);
|
var pageHeader = PageHeader.ReadFrom(buffer);
|
||||||
var isFreePage = freePageSet.Contains(pageId);
|
bool isFreePage = freePageSet.Contains(pageId);
|
||||||
|
|
||||||
int freeBytes = 0;
|
var freeBytes = 0;
|
||||||
if (isFreePage)
|
if (isFreePage)
|
||||||
{
|
|
||||||
freeBytes = _pageFile.PageSize;
|
freeBytes = _pageFile.PageSize;
|
||||||
}
|
else if (TryReadSlottedFreeSpace(buffer, out int slottedFreeBytes)) freeBytes = slottedFreeBytes;
|
||||||
else if (TryReadSlottedFreeSpace(buffer, out var slottedFreeBytes))
|
|
||||||
{
|
|
||||||
freeBytes = slottedFreeBytes;
|
|
||||||
}
|
|
||||||
|
|
||||||
totalFreeBytes += freeBytes;
|
totalFreeBytes += freeBytes;
|
||||||
|
|
||||||
@@ -378,7 +363,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
uint tailReclaimablePages = 0;
|
uint tailReclaimablePages = 0;
|
||||||
for (var i = pageCount; i > 2; i--)
|
for (uint i = pageCount; i > 2; i--)
|
||||||
{
|
{
|
||||||
if (!freePageSet.Contains(i - 1))
|
if (!freePageSet.Contains(i - 1))
|
||||||
break;
|
break;
|
||||||
@@ -386,12 +371,12 @@ public sealed partial class StorageEngine
|
|||||||
tailReclaimablePages++;
|
tailReclaimablePages++;
|
||||||
}
|
}
|
||||||
|
|
||||||
var fileBytes = Math.Max(1L, _pageFile.FileLengthBytes);
|
long fileBytes = Math.Max(1L, _pageFile.FileLengthBytes);
|
||||||
return new FragmentationMapReport
|
return new FragmentationMapReport
|
||||||
{
|
{
|
||||||
Pages = pages,
|
Pages = pages,
|
||||||
TotalFreeBytes = totalFreeBytes,
|
TotalFreeBytes = totalFreeBytes,
|
||||||
FragmentationPercent = (totalFreeBytes * 100d) / fileBytes,
|
FragmentationPercent = totalFreeBytes * 100d / fileBytes,
|
||||||
TailReclaimablePages = tailReclaimablePages
|
TailReclaimablePages = tailReclaimablePages
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -403,10 +388,8 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
var index = new BTreeIndex(this, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
var index = new BTreeIndex(this, IndexOptions.CreateUnique("_id"), metadata.PrimaryRootPageId);
|
||||||
|
|
||||||
foreach (var entry in index.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, transactionId: 0))
|
foreach (var entry in index.Range(IndexKey.MinKey, IndexKey.MaxKey, IndexDirection.Forward, 0))
|
||||||
{
|
|
||||||
yield return entry.Location;
|
yield return entry.Location;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool TryReadFirstOverflowPage(in DocumentLocation location, out uint firstOverflowPage)
|
private bool TryReadFirstOverflowPage(in DocumentLocation location, out uint firstOverflowPage)
|
||||||
@@ -419,7 +402,7 @@ public sealed partial class StorageEngine
|
|||||||
if (location.SlotIndex >= header.SlotCount)
|
if (location.SlotIndex >= header.SlotCount)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
int slotOffset = SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size;
|
||||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||||
return false;
|
return false;
|
||||||
@@ -441,7 +424,7 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
var buffer = new byte[_pageFile.PageSize];
|
var buffer = new byte[_pageFile.PageSize];
|
||||||
var visited = new HashSet<uint>();
|
var visited = new HashSet<uint>();
|
||||||
var current = firstOverflowPage;
|
uint current = firstOverflowPage;
|
||||||
|
|
||||||
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
||||||
{
|
{
|
||||||
@@ -472,12 +455,12 @@ public sealed partial class StorageEngine
|
|||||||
if (location.SlotIndex >= header.SlotCount)
|
if (location.SlotIndex >= header.SlotCount)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
int slotOffset = SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size;
|
||||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
bool hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||||
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
||||||
|
|
||||||
if (!hasOverflow)
|
if (!hasOverflow)
|
||||||
@@ -492,7 +475,8 @@ public sealed partial class StorageEngine
|
|||||||
if (slot.Length < CompressedPayloadHeader.Size)
|
if (slot.Length < CompressedPayloadHeader.Size)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var compressedHeader = CompressedPayloadHeader.ReadFrom(pageBuffer.AsSpan(slot.Offset, CompressedPayloadHeader.Size));
|
var compressedHeader =
|
||||||
|
CompressedPayloadHeader.ReadFrom(pageBuffer.AsSpan(slot.Offset, CompressedPayloadHeader.Size));
|
||||||
originalBytes = compressedHeader.OriginalLength;
|
originalBytes = compressedHeader.OriginalLength;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -501,7 +485,7 @@ public sealed partial class StorageEngine
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||||
var totalStoredBytes = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
int totalStoredBytes = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
||||||
if (totalStoredBytes < 0)
|
if (totalStoredBytes < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
@@ -522,8 +506,8 @@ public sealed partial class StorageEngine
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
storedPrefix.CopyTo(headerBuffer);
|
storedPrefix.CopyTo(headerBuffer);
|
||||||
var copied = storedPrefix.Length;
|
int copied = storedPrefix.Length;
|
||||||
var nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
uint nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
||||||
var overflowBuffer = new byte[_pageFile.PageSize];
|
var overflowBuffer = new byte[_pageFile.PageSize];
|
||||||
|
|
||||||
while (copied < CompressedPayloadHeader.Size && nextOverflow != 0 && nextOverflow < _pageFile.NextPageId)
|
while (copied < CompressedPayloadHeader.Size && nextOverflow != 0 && nextOverflow < _pageFile.NextPageId)
|
||||||
@@ -533,7 +517,8 @@ public sealed partial class StorageEngine
|
|||||||
if (overflowHeader.PageType != PageType.Overflow)
|
if (overflowHeader.PageType != PageType.Overflow)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var available = Math.Min(CompressedPayloadHeader.Size - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
int available = Math.Min(CompressedPayloadHeader.Size - copied,
|
||||||
|
_pageFile.PageSize - SlottedPageHeader.Size);
|
||||||
overflowBuffer.AsSpan(SlottedPageHeader.Size, available).CopyTo(headerBuffer.Slice(copied));
|
overflowBuffer.AsSpan(SlottedPageHeader.Size, available).CopyTo(headerBuffer.Slice(copied));
|
||||||
copied += available;
|
copied += available;
|
||||||
nextOverflow = overflowHeader.NextOverflowPage;
|
nextOverflow = overflowHeader.NextOverflowPage;
|
||||||
@@ -547,4 +532,4 @@ public sealed partial class StorageEngine
|
|||||||
originalBytes = headerFromPayload.OriginalLength;
|
originalBytes = headerFromPayload.OriginalLength;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,29 +1,104 @@
|
|||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
using System.Text;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
|
||||||
|
public sealed partial class StorageEngine
|
||||||
public sealed partial class StorageEngine
|
{
|
||||||
{
|
private readonly ConcurrentDictionary<string, ushort> _dictionaryCache = new(StringComparer.OrdinalIgnoreCase);
|
||||||
private readonly ConcurrentDictionary<string, ushort> _dictionaryCache = new(StringComparer.OrdinalIgnoreCase);
|
|
||||||
private readonly ConcurrentDictionary<ushort, string> _dictionaryReverseCache = new();
|
|
||||||
private uint _dictionaryRootPageId;
|
|
||||||
private ushort _nextDictionaryId;
|
|
||||||
|
|
||||||
// Lock for dictionary modifications (simple lock for now, could be RW lock)
|
// Lock for dictionary modifications (simple lock for now, could be RW lock)
|
||||||
private readonly object _dictionaryLock = new();
|
private readonly object _dictionaryLock = new();
|
||||||
|
private readonly ConcurrentDictionary<ushort, string> _dictionaryReverseCache = new();
|
||||||
private void InitializeDictionary()
|
private uint _dictionaryRootPageId;
|
||||||
{
|
private ushort _nextDictionaryId;
|
||||||
// 1. Read File Header (Page 0) to get Dictionary Root
|
|
||||||
var headerBuffer = new byte[PageSize];
|
/// <summary>
|
||||||
ReadPage(0, null, headerBuffer);
|
/// Gets the key-to-id dictionary cache.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The key-to-id map.</returns>
|
||||||
|
public ConcurrentDictionary<string, ushort> GetKeyMap()
|
||||||
|
{
|
||||||
|
return _dictionaryCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the id-to-key dictionary cache.
|
||||||
|
/// </summary>
|
||||||
|
/// <returns>The id-to-key map.</returns>
|
||||||
|
public ConcurrentDictionary<ushort, string> GetKeyReverseMap()
|
||||||
|
{
|
||||||
|
return _dictionaryReverseCache;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the ID for a dictionary key, creating it if it doesn't exist.
|
||||||
|
/// Thread-safe.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="key">The dictionary key.</param>
|
||||||
|
/// <returns>The dictionary identifier for the key.</returns>
|
||||||
|
public ushort GetOrAddDictionaryEntry(string key)
|
||||||
|
{
|
||||||
|
key = key.ToLowerInvariant();
|
||||||
|
if (_dictionaryCache.TryGetValue(key, out ushort id)) return id;
|
||||||
|
|
||||||
|
lock (_dictionaryLock)
|
||||||
|
{
|
||||||
|
// Double checked locking
|
||||||
|
if (_dictionaryCache.TryGetValue(key, out id)) return id;
|
||||||
|
|
||||||
|
// Try to find in storage (in case cache is incomplete or another process?)
|
||||||
|
// Note: FindAllGlobal loaded everything, so strict cache miss means it's not in DB.
|
||||||
|
// BUT if we support concurrent writers (multiple processed), we should re-check DB.
|
||||||
|
// Current CBDD seems to be single-process exclusive lock (FileShare.None).
|
||||||
|
// So in-memory cache is authoritative after load.
|
||||||
|
|
||||||
|
// Generate New ID
|
||||||
|
ushort nextId = _nextDictionaryId;
|
||||||
|
if (nextId == 0) nextId = DictionaryPage.ReservedValuesEnd + 1; // Should be init, but safety
|
||||||
|
|
||||||
|
// Insert into Page
|
||||||
|
// usage of default(ulong) or null transaction?
|
||||||
|
// Dictionary updates should ideally be transactional or immediate?
|
||||||
|
// "Immediate" for now to simplify, as dictionary is cross-collection.
|
||||||
|
// If we use transaction, we need to pass it in. For now, immediate write.
|
||||||
|
|
||||||
|
// We need to support "Insert Global" which handles overflow.
|
||||||
|
// DictionaryPage.Insert only handles single page.
|
||||||
|
|
||||||
|
// We need logic here to traverse chain and find space.
|
||||||
|
if (InsertDictionaryEntryGlobal(key, nextId))
|
||||||
|
{
|
||||||
|
_dictionaryCache[key] = nextId;
|
||||||
|
_dictionaryReverseCache[nextId] = key;
|
||||||
|
_nextDictionaryId++;
|
||||||
|
return nextId;
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new InvalidOperationException("Failed to insert dictionary entry (Storage Full?)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Registers a set of keys in the global dictionary.
|
||||||
|
/// Ensures all keys are assigned an ID and persisted.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="keys">The keys to register.</param>
|
||||||
|
public void RegisterKeys(IEnumerable<string> keys)
|
||||||
|
{
|
||||||
|
foreach (string key in keys) GetOrAddDictionaryEntry(key.ToLowerInvariant());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void InitializeDictionary()
|
||||||
|
{
|
||||||
|
// 1. Read File Header (Page 0) to get Dictionary Root
|
||||||
|
var headerBuffer = new byte[PageSize];
|
||||||
|
ReadPage(0, null, headerBuffer);
|
||||||
var header = PageHeader.ReadFrom(headerBuffer);
|
var header = PageHeader.ReadFrom(headerBuffer);
|
||||||
|
|
||||||
if (header.DictionaryRootPageId == 0)
|
if (header.DictionaryRootPageId == 0)
|
||||||
{
|
{
|
||||||
// Initialize new Dictionary
|
// Initialize new Dictionary
|
||||||
lock (_dictionaryLock)
|
lock (_dictionaryLock)
|
||||||
{
|
{
|
||||||
// Double check
|
// Double check
|
||||||
ReadPage(0, null, headerBuffer);
|
ReadPage(0, null, headerBuffer);
|
||||||
@@ -48,172 +123,92 @@ public sealed partial class StorageEngine
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
_dictionaryRootPageId = header.DictionaryRootPageId;
|
_dictionaryRootPageId = header.DictionaryRootPageId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
_dictionaryRootPageId = header.DictionaryRootPageId;
|
_dictionaryRootPageId = header.DictionaryRootPageId;
|
||||||
|
|
||||||
// Warm cache
|
// Warm cache
|
||||||
ushort maxId = DictionaryPage.ReservedValuesEnd;
|
ushort maxId = DictionaryPage.ReservedValuesEnd;
|
||||||
foreach (var (key, val) in DictionaryPage.FindAllGlobal(this, _dictionaryRootPageId))
|
foreach ((string key, ushort val) in DictionaryPage.FindAllGlobal(this, _dictionaryRootPageId))
|
||||||
{
|
{
|
||||||
var lowerKey = key.ToLowerInvariant();
|
string lowerKey = key.ToLowerInvariant();
|
||||||
_dictionaryCache[lowerKey] = val;
|
_dictionaryCache[lowerKey] = val;
|
||||||
_dictionaryReverseCache[val] = lowerKey;
|
_dictionaryReverseCache[val] = lowerKey;
|
||||||
if (val > maxId) maxId = val;
|
if (val > maxId) maxId = val;
|
||||||
}
|
}
|
||||||
_nextDictionaryId = (ushort)(maxId + 1);
|
|
||||||
}
|
_nextDictionaryId = (ushort)(maxId + 1);
|
||||||
|
}
|
||||||
// Pre-register internal keys used for Schema persistence
|
|
||||||
|
// Pre-register internal keys used for Schema persistence
|
||||||
RegisterKeys(new[] { "_id", "t", "_v", "f", "n", "b", "s", "a" });
|
RegisterKeys(new[] { "_id", "t", "_v", "f", "n", "b", "s", "a" });
|
||||||
|
|
||||||
// Pre-register common array indices to avoid mapping during high-frequency writes
|
// Pre-register common array indices to avoid mapping during high-frequency writes
|
||||||
var indices = new List<string>(101);
|
var indices = new List<string>(101);
|
||||||
for (int i = 0; i <= 100; i++) indices.Add(i.ToString());
|
for (var i = 0; i <= 100; i++) indices.Add(i.ToString());
|
||||||
RegisterKeys(indices);
|
RegisterKeys(indices);
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the key-to-id dictionary cache.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>The key-to-id map.</returns>
|
|
||||||
public ConcurrentDictionary<string, ushort> GetKeyMap() => _dictionaryCache;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the id-to-key dictionary cache.
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>The id-to-key map.</returns>
|
|
||||||
public ConcurrentDictionary<ushort, string> GetKeyReverseMap() => _dictionaryReverseCache;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the ID for a dictionary key, creating it if it doesn't exist.
|
|
||||||
/// Thread-safe.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="key">The dictionary key.</param>
|
|
||||||
/// <returns>The dictionary identifier for the key.</returns>
|
|
||||||
public ushort GetOrAddDictionaryEntry(string key)
|
|
||||||
{
|
|
||||||
key = key.ToLowerInvariant();
|
|
||||||
if (_dictionaryCache.TryGetValue(key, out var id))
|
|
||||||
{
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
lock (_dictionaryLock)
|
|
||||||
{
|
|
||||||
// Double checked locking
|
|
||||||
if (_dictionaryCache.TryGetValue(key, out id))
|
|
||||||
{
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to find in storage (in case cache is incomplete or another process?)
|
|
||||||
// Note: FindAllGlobal loaded everything, so strict cache miss means it's not in DB.
|
|
||||||
// BUT if we support concurrent writers (multiple processed), we should re-check DB.
|
|
||||||
// Current CBDD seems to be single-process exclusive lock (FileShare.None).
|
|
||||||
// So in-memory cache is authoritative after load.
|
|
||||||
|
|
||||||
// Generate New ID
|
|
||||||
ushort nextId = _nextDictionaryId;
|
|
||||||
if (nextId == 0) nextId = DictionaryPage.ReservedValuesEnd + 1; // Should be init, but safety
|
|
||||||
|
|
||||||
// Insert into Page
|
|
||||||
// usage of default(ulong) or null transaction?
|
|
||||||
// Dictionary updates should ideally be transactional or immediate?
|
|
||||||
// "Immediate" for now to simplify, as dictionary is cross-collection.
|
|
||||||
// If we use transaction, we need to pass it in. For now, immediate write.
|
|
||||||
|
|
||||||
// We need to support "Insert Global" which handles overflow.
|
|
||||||
// DictionaryPage.Insert only handles single page.
|
|
||||||
|
|
||||||
// We need logic here to traverse chain and find space.
|
|
||||||
if (InsertDictionaryEntryGlobal(key, nextId))
|
|
||||||
{
|
|
||||||
_dictionaryCache[key] = nextId;
|
|
||||||
_dictionaryReverseCache[nextId] = key;
|
|
||||||
_nextDictionaryId++;
|
|
||||||
return nextId;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw new InvalidOperationException("Failed to insert dictionary entry (Storage Full?)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the dictionary key for an identifier.
|
/// Gets the dictionary key for an identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="id">The dictionary identifier.</param>
|
/// <param name="id">The dictionary identifier.</param>
|
||||||
/// <returns>The dictionary key if found; otherwise, <see langword="null"/>.</returns>
|
/// <returns>The dictionary key if found; otherwise, <see langword="null" />.</returns>
|
||||||
public string? GetDictionaryKey(ushort id)
|
public string? GetDictionaryKey(ushort id)
|
||||||
{
|
{
|
||||||
if (_dictionaryReverseCache.TryGetValue(id, out var key))
|
if (_dictionaryReverseCache.TryGetValue(id, out string? key))
|
||||||
return key;
|
return key;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool InsertDictionaryEntryGlobal(string key, ushort value)
|
private bool InsertDictionaryEntryGlobal(string key, ushort value)
|
||||||
{
|
{
|
||||||
var pageId = _dictionaryRootPageId;
|
uint pageId = _dictionaryRootPageId;
|
||||||
var pageBuffer = new byte[PageSize];
|
var pageBuffer = new byte[PageSize];
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
ReadPage(pageId, null, pageBuffer);
|
ReadPage(pageId, null, pageBuffer);
|
||||||
|
|
||||||
// Try Insert
|
// Try Insert
|
||||||
if (DictionaryPage.Insert(pageBuffer, key, value))
|
if (DictionaryPage.Insert(pageBuffer, key, value))
|
||||||
{
|
{
|
||||||
// Success - Write Back
|
// Success - Write Back
|
||||||
WritePageImmediate(pageId, pageBuffer);
|
WritePageImmediate(pageId, pageBuffer);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Page Full - Check Next Page
|
// Page Full - Check Next Page
|
||||||
var header = PageHeader.ReadFrom(pageBuffer);
|
var header = PageHeader.ReadFrom(pageBuffer);
|
||||||
if (header.NextPageId != 0)
|
if (header.NextPageId != 0)
|
||||||
{
|
{
|
||||||
pageId = header.NextPageId;
|
pageId = header.NextPageId;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// No Next Page - Allocate New
|
// No Next Page - Allocate New
|
||||||
var newPageId = AllocatePage();
|
uint newPageId = AllocatePage();
|
||||||
var newPageBuffer = new byte[PageSize];
|
var newPageBuffer = new byte[PageSize];
|
||||||
DictionaryPage.Initialize(newPageBuffer, newPageId);
|
DictionaryPage.Initialize(newPageBuffer, newPageId);
|
||||||
|
|
||||||
// Should likely insert into NEW page immediately to save I/O?
|
// Should likely insert into NEW page immediately to save I/O?
|
||||||
// Or just link and loop?
|
// Or just link and loop?
|
||||||
// Let's Insert into new page logic here to avoid re-reading.
|
// Let's Insert into new page logic here to avoid re-reading.
|
||||||
if (!DictionaryPage.Insert(newPageBuffer, key, value))
|
if (!DictionaryPage.Insert(newPageBuffer, key, value))
|
||||||
return false; // Should not happen on empty page unless key is huge > page
|
return false; // Should not happen on empty page unless key is huge > page
|
||||||
|
|
||||||
// Write New Page
|
// Write New Page
|
||||||
WritePageImmediate(newPageId, newPageBuffer);
|
WritePageImmediate(newPageId, newPageBuffer);
|
||||||
|
|
||||||
// Update Previous Page Link
|
// Update Previous Page Link
|
||||||
header.NextPageId = newPageId;
|
header.NextPageId = newPageId;
|
||||||
header.WriteTo(pageBuffer);
|
header.WriteTo(pageBuffer);
|
||||||
WritePageImmediate(pageId, pageBuffer);
|
WritePageImmediate(pageId, pageBuffer);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
/// <summary>
|
|
||||||
/// Registers a set of keys in the global dictionary.
|
|
||||||
/// Ensures all keys are assigned an ID and persisted.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="keys">The keys to register.</param>
|
|
||||||
public void RegisterKeys(IEnumerable<string> keys)
|
|
||||||
{
|
|
||||||
foreach (var key in keys)
|
|
||||||
{
|
|
||||||
GetOrAddDictionaryEntry(key.ToLowerInvariant());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -15,31 +15,32 @@ internal readonly struct StorageFormatMetadata
|
|||||||
internal const int WireSize = 16;
|
internal const int WireSize = 16;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether format metadata is present.
|
/// Gets a value indicating whether format metadata is present.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool IsPresent { get; }
|
public bool IsPresent { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the storage format version.
|
/// Gets the storage format version.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public byte Version { get; }
|
public byte Version { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets enabled storage feature flags.
|
/// Gets enabled storage feature flags.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public StorageFeatureFlags FeatureFlags { get; }
|
public StorageFeatureFlags FeatureFlags { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the default compression codec.
|
/// Gets the default compression codec.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionCodec DefaultCodec { get; }
|
public CompressionCodec DefaultCodec { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether compression capability is enabled.
|
/// Gets a value indicating whether compression capability is enabled.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool CompressionCapabilityEnabled => (FeatureFlags & StorageFeatureFlags.CompressionCapability) != 0;
|
public bool CompressionCapabilityEnabled => (FeatureFlags & StorageFeatureFlags.CompressionCapability) != 0;
|
||||||
|
|
||||||
private StorageFormatMetadata(bool isPresent, byte version, StorageFeatureFlags featureFlags, CompressionCodec defaultCodec)
|
private StorageFormatMetadata(bool isPresent, byte version, StorageFeatureFlags featureFlags,
|
||||||
|
CompressionCodec defaultCodec)
|
||||||
{
|
{
|
||||||
IsPresent = isPresent;
|
IsPresent = isPresent;
|
||||||
Version = version;
|
Version = version;
|
||||||
@@ -48,18 +49,19 @@ internal readonly struct StorageFormatMetadata
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates metadata representing a modern format-aware file.
|
/// Creates metadata representing a modern format-aware file.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="version">The storage format version.</param>
|
/// <param name="version">The storage format version.</param>
|
||||||
/// <param name="featureFlags">Enabled feature flags.</param>
|
/// <param name="featureFlags">Enabled feature flags.</param>
|
||||||
/// <param name="defaultCodec">The default compression codec.</param>
|
/// <param name="defaultCodec">The default compression codec.</param>
|
||||||
public static StorageFormatMetadata Present(byte version, StorageFeatureFlags featureFlags, CompressionCodec defaultCodec)
|
public static StorageFormatMetadata Present(byte version, StorageFeatureFlags featureFlags,
|
||||||
|
CompressionCodec defaultCodec)
|
||||||
{
|
{
|
||||||
return new StorageFormatMetadata(true, version, featureFlags, defaultCodec);
|
return new StorageFormatMetadata(true, version, featureFlags, defaultCodec);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Creates metadata representing a legacy file without format metadata.
|
/// Creates metadata representing a legacy file without format metadata.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="defaultCodec">The default compression codec.</param>
|
/// <param name="defaultCodec">The default compression codec.</param>
|
||||||
public static StorageFormatMetadata Legacy(CompressionCodec defaultCodec)
|
public static StorageFormatMetadata Legacy(CompressionCodec defaultCodec)
|
||||||
@@ -88,12 +90,13 @@ public sealed partial class StorageEngine
|
|||||||
return metadata;
|
return metadata;
|
||||||
|
|
||||||
if (!_pageFile.WasCreated)
|
if (!_pageFile.WasCreated)
|
||||||
return StorageFormatMetadata.Legacy(_compressionOptions.Codec);
|
return StorageFormatMetadata.Legacy(CompressionOptions.Codec);
|
||||||
|
|
||||||
var featureFlags = _compressionOptions.EnableCompression
|
var featureFlags = CompressionOptions.EnableCompression
|
||||||
? StorageFeatureFlags.CompressionCapability
|
? StorageFeatureFlags.CompressionCapability
|
||||||
: StorageFeatureFlags.None;
|
: StorageFeatureFlags.None;
|
||||||
var initialMetadata = StorageFormatMetadata.Present(CurrentStorageFormatVersion, featureFlags, _compressionOptions.Codec);
|
var initialMetadata =
|
||||||
|
StorageFormatMetadata.Present(CurrentStorageFormatVersion, featureFlags, CompressionOptions.Codec);
|
||||||
WriteStorageFormatMetadata(initialMetadata);
|
WriteStorageFormatMetadata(initialMetadata);
|
||||||
return initialMetadata;
|
return initialMetadata;
|
||||||
}
|
}
|
||||||
@@ -104,11 +107,11 @@ public sealed partial class StorageEngine
|
|||||||
if (source.Length < StorageFormatMetadata.WireSize)
|
if (source.Length < StorageFormatMetadata.WireSize)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var magic = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(0, 4));
|
uint magic = BinaryPrimitives.ReadUInt32LittleEndian(source.Slice(0, 4));
|
||||||
if (magic != StorageFormatMagic)
|
if (magic != StorageFormatMagic)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var version = source[4];
|
byte version = source[4];
|
||||||
var featureFlags = (StorageFeatureFlags)source[5];
|
var featureFlags = (StorageFeatureFlags)source[5];
|
||||||
var codec = (CompressionCodec)source[6];
|
var codec = (CompressionCodec)source[6];
|
||||||
if (!Enum.IsDefined(codec))
|
if (!Enum.IsDefined(codec))
|
||||||
@@ -128,4 +131,4 @@ public sealed partial class StorageEngine
|
|||||||
buffer[6] = (byte)metadata.DefaultCodec;
|
buffer[6] = (byte)metadata.DefaultCodec;
|
||||||
_pageFile.WritePageZeroExtension(StorageHeaderExtensionOffset, buffer);
|
_pageFile.WritePageZeroExtension(StorageHeaderExtensionOffset, buffer);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,11 +1,9 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Allocates a new page.
|
/// Allocates a new page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>Page ID of the allocated page</returns>
|
/// <returns>Page ID of the allocated page</returns>
|
||||||
public uint AllocatePage()
|
public uint AllocatePage()
|
||||||
@@ -14,11 +12,11 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Frees a page.
|
/// Frees a page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">Page to free</param>
|
/// <param name="pageId">Page to free</param>
|
||||||
public void FreePage(uint pageId)
|
public void FreePage(uint pageId)
|
||||||
{
|
{
|
||||||
_pageFile.FreePage(pageId);
|
_pageFile.FreePage(pageId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -5,98 +5,98 @@ using ZB.MOM.WW.CBDD.Core.Compression;
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Options controlling compression migration.
|
/// Options controlling compression migration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompressionMigrationOptions
|
public sealed class CompressionMigrationOptions
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Enables dry-run estimation without mutating database contents.
|
/// Enables dry-run estimation without mutating database contents.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool DryRun { get; init; } = true;
|
public bool DryRun { get; init; } = true;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Target codec for migrated payloads.
|
/// Target codec for migrated payloads.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
public CompressionCodec Codec { get; init; } = CompressionCodec.Brotli;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Target compression level.
|
/// Target compression level.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
public CompressionLevel Level { get; init; } = CompressionLevel.Fastest;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Minimum logical payload size required before compression is attempted.
|
/// Minimum logical payload size required before compression is attempted.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int MinSizeBytes { get; init; } = 1024;
|
public int MinSizeBytes { get; init; } = 1024;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Minimum savings percent required to keep compressed output.
|
/// Minimum savings percent required to keep compressed output.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int MinSavingsPercent { get; init; } = 10;
|
public int MinSavingsPercent { get; init; } = 10;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Optional include-only collection list (case-insensitive).
|
/// Optional include-only collection list (case-insensitive).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<string>? IncludeCollections { get; init; }
|
public IReadOnlyList<string>? IncludeCollections { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Optional exclusion collection list (case-insensitive).
|
/// Optional exclusion collection list (case-insensitive).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public IReadOnlyList<string>? ExcludeCollections { get; init; }
|
public IReadOnlyList<string>? ExcludeCollections { get; init; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Result of a compression migration run.
|
/// Result of a compression migration run.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class CompressionMigrationResult
|
public sealed class CompressionMigrationResult
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets a value indicating whether this run was executed in dry-run mode.
|
/// Gets a value indicating whether this run was executed in dry-run mode.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool DryRun { get; init; }
|
public bool DryRun { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the target codec used for migration output.
|
/// Gets the target codec used for migration output.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionCodec Codec { get; init; }
|
public CompressionCodec Codec { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the target compression level used for migration output.
|
/// Gets the target compression level used for migration output.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public CompressionLevel Level { get; init; }
|
public CompressionLevel Level { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of collections processed.
|
/// Gets the number of collections processed.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int CollectionsProcessed { get; init; }
|
public int CollectionsProcessed { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of documents scanned.
|
/// Gets the number of documents scanned.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DocumentsScanned { get; init; }
|
public long DocumentsScanned { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of documents rewritten.
|
/// Gets the number of documents rewritten.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DocumentsRewritten { get; init; }
|
public long DocumentsRewritten { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of documents skipped.
|
/// Gets the number of documents skipped.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long DocumentsSkipped { get; init; }
|
public long DocumentsSkipped { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the total logical bytes observed before migration decisions.
|
/// Gets the total logical bytes observed before migration decisions.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesBefore { get; init; }
|
public long BytesBefore { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the estimated total stored bytes after migration.
|
/// Gets the estimated total stored bytes after migration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesEstimatedAfter { get; init; }
|
public long BytesEstimatedAfter { get; init; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the actual total stored bytes after migration when not in dry-run mode.
|
/// Gets the actual total stored bytes after migration when not in dry-run mode.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long BytesActualAfter { get; init; }
|
public long BytesActualAfter { get; init; }
|
||||||
}
|
}
|
||||||
@@ -104,7 +104,7 @@ public sealed class CompressionMigrationResult
|
|||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Estimates or applies a one-time compression migration.
|
/// Estimates or applies a one-time compression migration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Optional compression migration options.</param>
|
/// <param name="options">Optional compression migration options.</param>
|
||||||
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
public CompressionMigrationResult MigrateCompression(CompressionMigrationOptions? options = null)
|
||||||
@@ -113,11 +113,12 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Estimates or applies a one-time compression migration.
|
/// Estimates or applies a one-time compression migration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="options">Optional compression migration options.</param>
|
/// <param name="options">Optional compression migration options.</param>
|
||||||
/// <param name="ct">A token used to cancel the operation.</param>
|
/// <param name="ct">A token used to cancel the operation.</param>
|
||||||
public async Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null, CancellationToken ct = default)
|
public async Task<CompressionMigrationResult> MigrateCompressionAsync(CompressionMigrationOptions? options = null,
|
||||||
|
CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
var normalized = NormalizeMigrationOptions(options);
|
var normalized = NormalizeMigrationOptions(options);
|
||||||
|
|
||||||
@@ -147,13 +148,13 @@ public sealed partial class StorageEngine
|
|||||||
{
|
{
|
||||||
ct.ThrowIfCancellationRequested();
|
ct.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
if (!TryReadStoredPayload(location, out var storedPayload, out var isCompressed))
|
if (!TryReadStoredPayload(location, out byte[] storedPayload, out bool isCompressed))
|
||||||
{
|
{
|
||||||
docsSkipped++;
|
docsSkipped++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!TryGetLogicalPayload(storedPayload, isCompressed, out var logicalPayload))
|
if (!TryGetLogicalPayload(storedPayload, isCompressed, out byte[] logicalPayload))
|
||||||
{
|
{
|
||||||
docsSkipped++;
|
docsSkipped++;
|
||||||
continue;
|
continue;
|
||||||
@@ -162,15 +163,14 @@ public sealed partial class StorageEngine
|
|||||||
docsScanned++;
|
docsScanned++;
|
||||||
bytesBefore += logicalPayload.Length;
|
bytesBefore += logicalPayload.Length;
|
||||||
|
|
||||||
var targetStored = BuildTargetStoredPayload(logicalPayload, normalized, out var targetCompressed);
|
byte[] targetStored =
|
||||||
|
BuildTargetStoredPayload(logicalPayload, normalized, out bool targetCompressed);
|
||||||
bytesEstimatedAfter += targetStored.Length;
|
bytesEstimatedAfter += targetStored.Length;
|
||||||
|
|
||||||
if (normalized.DryRun)
|
if (normalized.DryRun) continue;
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!TryRewriteStoredPayloadAtLocation(location, targetStored, targetCompressed, out var actualStoredBytes))
|
if (!TryRewriteStoredPayloadAtLocation(location, targetStored, targetCompressed,
|
||||||
|
out int actualStoredBytes))
|
||||||
{
|
{
|
||||||
docsSkipped++;
|
docsSkipped++;
|
||||||
continue;
|
continue;
|
||||||
@@ -184,9 +184,9 @@ public sealed partial class StorageEngine
|
|||||||
if (!normalized.DryRun)
|
if (!normalized.DryRun)
|
||||||
{
|
{
|
||||||
var metadata = StorageFormatMetadata.Present(
|
var metadata = StorageFormatMetadata.Present(
|
||||||
version: 1,
|
1,
|
||||||
featureFlags: StorageFeatureFlags.CompressionCapability,
|
StorageFeatureFlags.CompressionCapability,
|
||||||
defaultCodec: normalized.Codec);
|
normalized.Codec);
|
||||||
WriteStorageFormatMetadata(metadata);
|
WriteStorageFormatMetadata(metadata);
|
||||||
_pageFile.Flush();
|
_pageFile.Flush();
|
||||||
}
|
}
|
||||||
@@ -221,7 +221,8 @@ public sealed partial class StorageEngine
|
|||||||
var normalized = options ?? new CompressionMigrationOptions();
|
var normalized = options ?? new CompressionMigrationOptions();
|
||||||
|
|
||||||
if (!Enum.IsDefined(normalized.Codec) || normalized.Codec == CompressionCodec.None)
|
if (!Enum.IsDefined(normalized.Codec) || normalized.Codec == CompressionCodec.None)
|
||||||
throw new ArgumentOutOfRangeException(nameof(options), "Migration codec must be a supported non-None codec.");
|
throw new ArgumentOutOfRangeException(nameof(options),
|
||||||
|
"Migration codec must be a supported non-None codec.");
|
||||||
|
|
||||||
if (normalized.MinSizeBytes < 0)
|
if (normalized.MinSizeBytes < 0)
|
||||||
throw new ArgumentOutOfRangeException(nameof(options), "MinSizeBytes must be non-negative.");
|
throw new ArgumentOutOfRangeException(nameof(options), "MinSizeBytes must be non-negative.");
|
||||||
@@ -250,7 +251,8 @@ public sealed partial class StorageEngine
|
|||||||
.ToList();
|
.ToList();
|
||||||
}
|
}
|
||||||
|
|
||||||
private byte[] BuildTargetStoredPayload(ReadOnlySpan<byte> logicalPayload, CompressionMigrationOptions options, out bool compressed)
|
private byte[] BuildTargetStoredPayload(ReadOnlySpan<byte> logicalPayload, CompressionMigrationOptions options,
|
||||||
|
out bool compressed)
|
||||||
{
|
{
|
||||||
compressed = false;
|
compressed = false;
|
||||||
|
|
||||||
@@ -259,10 +261,10 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
var compressedPayload = _compressionService.Compress(logicalPayload, options.Codec, options.Level);
|
byte[] compressedPayload = CompressionService.Compress(logicalPayload, options.Codec, options.Level);
|
||||||
var storedLength = CompressedPayloadHeader.Size + compressedPayload.Length;
|
int storedLength = CompressedPayloadHeader.Size + compressedPayload.Length;
|
||||||
var savings = logicalPayload.Length - storedLength;
|
int savings = logicalPayload.Length - storedLength;
|
||||||
var savingsPercent = logicalPayload.Length == 0 ? 0 : (int)((savings * 100L) / logicalPayload.Length);
|
int savingsPercent = logicalPayload.Length == 0 ? 0 : (int)(savings * 100L / logicalPayload.Length);
|
||||||
if (savings <= 0 || savingsPercent < options.MinSavingsPercent)
|
if (savings <= 0 || savingsPercent < options.MinSavingsPercent)
|
||||||
return logicalPayload.ToArray();
|
return logicalPayload.ToArray();
|
||||||
|
|
||||||
@@ -308,11 +310,11 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
logicalPayload = _compressionService.Decompress(
|
logicalPayload = CompressionService.Decompress(
|
||||||
compressedPayload,
|
compressedPayload,
|
||||||
header.Codec,
|
header.Codec,
|
||||||
header.OriginalLength,
|
header.OriginalLength,
|
||||||
Math.Max(header.OriginalLength, _compressionOptions.MaxDecompressedSizeBytes));
|
Math.Max(header.OriginalLength, CompressionOptions.MaxDecompressedSizeBytes));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
catch
|
catch
|
||||||
@@ -336,13 +338,13 @@ public sealed partial class StorageEngine
|
|||||||
if (location.SlotIndex >= header.SlotCount)
|
if (location.SlotIndex >= header.SlotCount)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
int slotOffset = SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size;
|
||||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
isCompressed = (slot.Flags & SlotFlags.Compressed) != 0;
|
||||||
var hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
bool hasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||||
|
|
||||||
if (!hasOverflow)
|
if (!hasOverflow)
|
||||||
{
|
{
|
||||||
@@ -354,14 +356,14 @@ public sealed partial class StorageEngine
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
var primaryPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||||
var totalStoredLength = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
int totalStoredLength = BinaryPrimitives.ReadInt32LittleEndian(primaryPayload.Slice(0, 4));
|
||||||
var nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
uint nextOverflow = BinaryPrimitives.ReadUInt32LittleEndian(primaryPayload.Slice(4, 4));
|
||||||
if (totalStoredLength < 0)
|
if (totalStoredLength < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var output = new byte[totalStoredLength];
|
var output = new byte[totalStoredLength];
|
||||||
var primaryChunk = primaryPayload.Slice(8);
|
var primaryChunk = primaryPayload.Slice(8);
|
||||||
var copied = Math.Min(primaryChunk.Length, output.Length);
|
int copied = Math.Min(primaryChunk.Length, output.Length);
|
||||||
primaryChunk.Slice(0, copied).CopyTo(output);
|
primaryChunk.Slice(0, copied).CopyTo(output);
|
||||||
|
|
||||||
var overflowBuffer = new byte[_pageFile.PageSize];
|
var overflowBuffer = new byte[_pageFile.PageSize];
|
||||||
@@ -372,7 +374,7 @@ public sealed partial class StorageEngine
|
|||||||
if (overflowHeader.PageType != PageType.Overflow)
|
if (overflowHeader.PageType != PageType.Overflow)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var chunk = Math.Min(output.Length - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
int chunk = Math.Min(output.Length - copied, _pageFile.PageSize - SlottedPageHeader.Size);
|
||||||
overflowBuffer.AsSpan(SlottedPageHeader.Size, chunk).CopyTo(output.AsSpan(copied));
|
overflowBuffer.AsSpan(SlottedPageHeader.Size, chunk).CopyTo(output.AsSpan(copied));
|
||||||
copied += chunk;
|
copied += chunk;
|
||||||
nextOverflow = overflowHeader.NextOverflowPage;
|
nextOverflow = overflowHeader.NextOverflowPage;
|
||||||
@@ -403,12 +405,12 @@ public sealed partial class StorageEngine
|
|||||||
if (location.SlotIndex >= pageHeader.SlotCount)
|
if (location.SlotIndex >= pageHeader.SlotCount)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var slotOffset = SlottedPageHeader.Size + (location.SlotIndex * SlotEntry.Size);
|
int slotOffset = SlottedPageHeader.Size + location.SlotIndex * SlotEntry.Size;
|
||||||
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
var slot = SlotEntry.ReadFrom(pageBuffer.AsSpan(slotOffset, SlotEntry.Size));
|
||||||
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
if ((slot.Flags & SlotFlags.Deleted) != 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var oldHasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
bool oldHasOverflow = (slot.Flags & SlotFlags.HasOverflow) != 0;
|
||||||
uint oldOverflowHead = 0;
|
uint oldOverflowHead = 0;
|
||||||
if (oldHasOverflow)
|
if (oldHasOverflow)
|
||||||
{
|
{
|
||||||
@@ -442,12 +444,12 @@ public sealed partial class StorageEngine
|
|||||||
if (slot.Length < 8)
|
if (slot.Length < 8)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var primaryChunkSize = slot.Length - 8;
|
int primaryChunkSize = slot.Length - 8;
|
||||||
if (primaryChunkSize < 0)
|
if (primaryChunkSize < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
var remainder = newStoredPayload.Slice(primaryChunkSize);
|
var remainder = newStoredPayload.Slice(primaryChunkSize);
|
||||||
var newOverflowHead = BuildOverflowChainForMigration(remainder);
|
uint newOverflowHead = BuildOverflowChainForMigration(remainder);
|
||||||
|
|
||||||
var slotPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
var slotPayload = pageBuffer.AsSpan(slot.Offset, slot.Length);
|
||||||
slotPayload.Clear();
|
slotPayload.Clear();
|
||||||
@@ -475,22 +477,22 @@ public sealed partial class StorageEngine
|
|||||||
if (overflowPayload.IsEmpty)
|
if (overflowPayload.IsEmpty)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
var chunkSize = _pageFile.PageSize - SlottedPageHeader.Size;
|
int chunkSize = _pageFile.PageSize - SlottedPageHeader.Size;
|
||||||
uint nextOverflowPageId = 0;
|
uint nextOverflowPageId = 0;
|
||||||
|
|
||||||
var tailSize = overflowPayload.Length % chunkSize;
|
int tailSize = overflowPayload.Length % chunkSize;
|
||||||
var fullPages = overflowPayload.Length / chunkSize;
|
int fullPages = overflowPayload.Length / chunkSize;
|
||||||
|
|
||||||
if (tailSize > 0)
|
if (tailSize > 0)
|
||||||
{
|
{
|
||||||
var tailOffset = fullPages * chunkSize;
|
int tailOffset = fullPages * chunkSize;
|
||||||
var tailSlice = overflowPayload.Slice(tailOffset, tailSize);
|
var tailSlice = overflowPayload.Slice(tailOffset, tailSize);
|
||||||
nextOverflowPageId = AllocateOverflowPageForMigration(tailSlice, nextOverflowPageId);
|
nextOverflowPageId = AllocateOverflowPageForMigration(tailSlice, nextOverflowPageId);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (var i = fullPages - 1; i >= 0; i--)
|
for (int i = fullPages - 1; i >= 0; i--)
|
||||||
{
|
{
|
||||||
var chunkOffset = i * chunkSize;
|
int chunkOffset = i * chunkSize;
|
||||||
var chunk = overflowPayload.Slice(chunkOffset, chunkSize);
|
var chunk = overflowPayload.Slice(chunkOffset, chunkSize);
|
||||||
nextOverflowPageId = AllocateOverflowPageForMigration(chunk, nextOverflowPageId);
|
nextOverflowPageId = AllocateOverflowPageForMigration(chunk, nextOverflowPageId);
|
||||||
}
|
}
|
||||||
@@ -500,7 +502,7 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
private uint AllocateOverflowPageForMigration(ReadOnlySpan<byte> payloadChunk, uint nextOverflowPageId)
|
private uint AllocateOverflowPageForMigration(ReadOnlySpan<byte> payloadChunk, uint nextOverflowPageId)
|
||||||
{
|
{
|
||||||
var pageId = _pageFile.AllocatePage();
|
uint pageId = _pageFile.AllocatePage();
|
||||||
var buffer = new byte[_pageFile.PageSize];
|
var buffer = new byte[_pageFile.PageSize];
|
||||||
|
|
||||||
var header = new SlottedPageHeader
|
var header = new SlottedPageHeader
|
||||||
@@ -524,15 +526,15 @@ public sealed partial class StorageEngine
|
|||||||
{
|
{
|
||||||
var buffer = new byte[_pageFile.PageSize];
|
var buffer = new byte[_pageFile.PageSize];
|
||||||
var visited = new HashSet<uint>();
|
var visited = new HashSet<uint>();
|
||||||
var current = firstOverflowPage;
|
uint current = firstOverflowPage;
|
||||||
|
|
||||||
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
while (current != 0 && current < _pageFile.NextPageId && visited.Add(current))
|
||||||
{
|
{
|
||||||
_pageFile.ReadPage(current, buffer);
|
_pageFile.ReadPage(current, buffer);
|
||||||
var header = SlottedPageHeader.ReadFrom(buffer);
|
var header = SlottedPageHeader.ReadFrom(buffer);
|
||||||
var next = header.NextOverflowPage;
|
uint next = header.NextOverflowPage;
|
||||||
_pageFile.FreePage(current);
|
_pageFile.FreePage(current);
|
||||||
current = next;
|
current = next;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,14 +1,14 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using System.Collections.Concurrent;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads a page with transaction isolation.
|
/// Reads a page with transaction isolation.
|
||||||
/// 1. Check WAL cache for uncommitted writes (Read Your Own Writes)
|
/// 1. Check WAL cache for uncommitted writes (Read Your Own Writes)
|
||||||
/// 2. Check WAL index for committed writes (lazy replay)
|
/// 2. Check WAL index for committed writes (lazy replay)
|
||||||
/// 3. Read from PageFile (committed baseline)
|
/// 3. Read from PageFile (committed baseline)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">Page to read</param>
|
/// <param name="pageId">Page to read</param>
|
||||||
/// <param name="transactionId">Optional transaction ID for isolation</param>
|
/// <param name="transactionId">Optional transaction ID for isolation</param>
|
||||||
@@ -17,32 +17,32 @@ public sealed partial class StorageEngine
|
|||||||
{
|
{
|
||||||
// 1. Check transaction-local WAL cache (Read Your Own Writes)
|
// 1. Check transaction-local WAL cache (Read Your Own Writes)
|
||||||
// transactionId=0 or null means "no active transaction, read committed only"
|
// transactionId=0 or null means "no active transaction, read committed only"
|
||||||
if (transactionId.HasValue &&
|
if (transactionId.HasValue &&
|
||||||
transactionId.Value != 0 &&
|
transactionId.Value != 0 &&
|
||||||
_walCache.TryGetValue(transactionId.Value, out var txnPages) &&
|
_walCache.TryGetValue(transactionId.Value, out var txnPages) &&
|
||||||
txnPages.TryGetValue(pageId, out var uncommittedData))
|
txnPages.TryGetValue(pageId, out byte[]? uncommittedData))
|
||||||
{
|
{
|
||||||
var length = Math.Min(uncommittedData.Length, destination.Length);
|
int length = Math.Min(uncommittedData.Length, destination.Length);
|
||||||
uncommittedData.AsSpan(0, length).CopyTo(destination);
|
uncommittedData.AsSpan(0, length).CopyTo(destination);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Check WAL index (committed but not checkpointed)
|
// 2. Check WAL index (committed but not checkpointed)
|
||||||
if (_walIndex.TryGetValue(pageId, out var committedData))
|
if (_walIndex.TryGetValue(pageId, out byte[]? committedData))
|
||||||
{
|
{
|
||||||
var length = Math.Min(committedData.Length, destination.Length);
|
int length = Math.Min(committedData.Length, destination.Length);
|
||||||
committedData.AsSpan(0, length).CopyTo(destination);
|
committedData.AsSpan(0, length).CopyTo(destination);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Read committed baseline from PageFile
|
// 3. Read committed baseline from PageFile
|
||||||
_pageFile.ReadPage(pageId, destination);
|
_pageFile.ReadPage(pageId, destination);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a page within a transaction.
|
/// Writes a page within a transaction.
|
||||||
/// Data goes to WAL cache immediately and becomes visible to that transaction only.
|
/// Data goes to WAL cache immediately and becomes visible to that transaction only.
|
||||||
/// Will be written to WAL on commit.
|
/// Will be written to WAL on commit.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">Page to write</param>
|
/// <param name="pageId">Page to write</param>
|
||||||
/// <param name="transactionId">Transaction ID owning this write</param>
|
/// <param name="transactionId">Transaction ID owning this write</param>
|
||||||
@@ -50,20 +50,20 @@ public sealed partial class StorageEngine
|
|||||||
public void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data)
|
public void WritePage(uint pageId, ulong transactionId, ReadOnlySpan<byte> data)
|
||||||
{
|
{
|
||||||
if (transactionId == 0)
|
if (transactionId == 0)
|
||||||
throw new InvalidOperationException("Cannot write without a transaction (transactionId=0 is reserved)");
|
throw new InvalidOperationException("Cannot write without a transaction (transactionId=0 is reserved)");
|
||||||
|
|
||||||
// Get or create transaction-local cache
|
// Get or create transaction-local cache
|
||||||
var txnPages = _walCache.GetOrAdd(transactionId,
|
var txnPages = _walCache.GetOrAdd(transactionId,
|
||||||
_ => new System.Collections.Concurrent.ConcurrentDictionary<uint, byte[]>());
|
_ => new ConcurrentDictionary<uint, byte[]>());
|
||||||
|
|
||||||
// Store defensive copy
|
// Store defensive copy
|
||||||
var copy = data.ToArray();
|
byte[] copy = data.ToArray();
|
||||||
txnPages[pageId] = copy;
|
txnPages[pageId] = copy;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a page immediately to disk (non-transactional).
|
/// Writes a page immediately to disk (non-transactional).
|
||||||
/// Used for initialization and metadata updates outside of transactions.
|
/// Used for initialization and metadata updates outside of transactions.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">Page to write</param>
|
/// <param name="pageId">Page to write</param>
|
||||||
/// <param name="data">Page data</param>
|
/// <param name="data">Page data</param>
|
||||||
@@ -73,8 +73,8 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of pages currently allocated in the page file.
|
/// Gets the number of pages currently allocated in the page file.
|
||||||
/// Useful for full database scans.
|
/// Useful for full database scans.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageCount => _pageFile.NextPageId;
|
public uint PageCount => _pageFile.NextPageId;
|
||||||
}
|
}
|
||||||
@@ -1,36 +1,36 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current size of the WAL file.
|
/// Gets the current size of the WAL file.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public long GetWalSize()
|
public long GetWalSize()
|
||||||
{
|
{
|
||||||
return _wal.GetCurrentSize();
|
return _wal.GetCurrentSize();
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Truncates the WAL file.
|
|
||||||
/// Should only be called after a successful checkpoint.
|
|
||||||
/// </summary>
|
|
||||||
public void TruncateWal()
|
|
||||||
{
|
|
||||||
_wal.Truncate();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Flushes the WAL to disk.
|
|
||||||
/// </summary>
|
|
||||||
public void FlushWal()
|
|
||||||
{
|
|
||||||
_wal.Flush();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a truncate checkpoint by default.
|
/// Truncates the WAL file.
|
||||||
|
/// Should only be called after a successful checkpoint.
|
||||||
|
/// </summary>
|
||||||
|
public void TruncateWal()
|
||||||
|
{
|
||||||
|
_wal.Truncate();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Flushes the WAL to disk.
|
||||||
|
/// </summary>
|
||||||
|
public void FlushWal()
|
||||||
|
{
|
||||||
|
_wal.Flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Performs a truncate checkpoint by default.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Checkpoint()
|
public void Checkpoint()
|
||||||
{
|
{
|
||||||
@@ -38,7 +38,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a checkpoint using the requested mode.
|
/// Performs a checkpoint using the requested mode.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
/// <returns>The checkpoint execution result.</returns>
|
/// <returns>The checkpoint execution result.</returns>
|
||||||
@@ -50,7 +50,7 @@ public sealed partial class StorageEngine
|
|||||||
lockAcquired = _commitLock.Wait(0);
|
lockAcquired = _commitLock.Wait(0);
|
||||||
if (!lockAcquired)
|
if (!lockAcquired)
|
||||||
{
|
{
|
||||||
var walSize = _wal.GetCurrentSize();
|
long walSize = _wal.GetCurrentSize();
|
||||||
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,19 +66,18 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
if (lockAcquired)
|
if (lockAcquired) _commitLock.Release();
|
||||||
{
|
|
||||||
_commitLock.Release();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void CheckpointInternal()
|
private void CheckpointInternal()
|
||||||
=> _ = CheckpointInternal(CheckpointMode.Truncate);
|
{
|
||||||
|
_ = CheckpointInternal(CheckpointMode.Truncate);
|
||||||
|
}
|
||||||
|
|
||||||
private CheckpointResult CheckpointInternal(CheckpointMode mode)
|
private CheckpointResult CheckpointInternal(CheckpointMode mode)
|
||||||
{
|
{
|
||||||
var walBytesBefore = _wal.GetCurrentSize();
|
long walBytesBefore = _wal.GetCurrentSize();
|
||||||
var appliedPages = 0;
|
var appliedPages = 0;
|
||||||
var truncated = false;
|
var truncated = false;
|
||||||
var restarted = false;
|
var restarted = false;
|
||||||
@@ -91,10 +90,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 2. Flush PageFile to ensure durability.
|
// 2. Flush PageFile to ensure durability.
|
||||||
if (appliedPages > 0)
|
if (appliedPages > 0) _pageFile.Flush();
|
||||||
{
|
|
||||||
_pageFile.Flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Clear in-memory WAL index (now persisted).
|
// 3. Clear in-memory WAL index (now persisted).
|
||||||
_walIndex.Clear();
|
_walIndex.Clear();
|
||||||
@@ -109,6 +105,7 @@ public sealed partial class StorageEngine
|
|||||||
_wal.WriteCheckpointRecord();
|
_wal.WriteCheckpointRecord();
|
||||||
_wal.Flush();
|
_wal.Flush();
|
||||||
}
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
case CheckpointMode.Truncate:
|
case CheckpointMode.Truncate:
|
||||||
if (walBytesBefore > 0)
|
if (walBytesBefore > 0)
|
||||||
@@ -116,6 +113,7 @@ public sealed partial class StorageEngine
|
|||||||
_wal.Truncate();
|
_wal.Truncate();
|
||||||
truncated = true;
|
truncated = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
case CheckpointMode.Restart:
|
case CheckpointMode.Restart:
|
||||||
_wal.Restart();
|
_wal.Restart();
|
||||||
@@ -126,12 +124,12 @@ public sealed partial class StorageEngine
|
|||||||
throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported checkpoint mode.");
|
throw new ArgumentOutOfRangeException(nameof(mode), mode, "Unsupported checkpoint mode.");
|
||||||
}
|
}
|
||||||
|
|
||||||
var walBytesAfter = _wal.GetCurrentSize();
|
long walBytesAfter = _wal.GetCurrentSize();
|
||||||
return new CheckpointResult(mode, true, appliedPages, walBytesBefore, walBytesAfter, truncated, restarted);
|
return new CheckpointResult(mode, true, appliedPages, walBytesBefore, walBytesAfter, truncated, restarted);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a truncate checkpoint asynchronously by default.
|
/// Performs a truncate checkpoint asynchronously by default.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
public async Task CheckpointAsync(CancellationToken ct = default)
|
public async Task CheckpointAsync(CancellationToken ct = default)
|
||||||
@@ -140,7 +138,7 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Performs a checkpoint asynchronously using the requested mode.
|
/// Performs a checkpoint asynchronously using the requested mode.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="mode">Checkpoint mode to execute.</param>
|
/// <param name="mode">Checkpoint mode to execute.</param>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
@@ -153,7 +151,7 @@ public sealed partial class StorageEngine
|
|||||||
lockAcquired = await _commitLock.WaitAsync(0, ct);
|
lockAcquired = await _commitLock.WaitAsync(0, ct);
|
||||||
if (!lockAcquired)
|
if (!lockAcquired)
|
||||||
{
|
{
|
||||||
var walSize = _wal.GetCurrentSize();
|
long walSize = _wal.GetCurrentSize();
|
||||||
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
return new CheckpointResult(mode, false, 0, walSize, walSize, false, false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -170,16 +168,13 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
if (lockAcquired)
|
if (lockAcquired) _commitLock.Release();
|
||||||
{
|
|
||||||
_commitLock.Release();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Recovers from crash by replaying WAL.
|
/// Recovers from crash by replaying WAL.
|
||||||
/// Applies committed transactions to PageFile in deterministic WAL order, then truncates WAL.
|
/// Applies committed transactions to PageFile in deterministic WAL order, then truncates WAL.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Recover()
|
public void Recover()
|
||||||
{
|
{
|
||||||
@@ -189,35 +184,28 @@ public sealed partial class StorageEngine
|
|||||||
// 1. Read WAL and locate the latest checkpoint boundary.
|
// 1. Read WAL and locate the latest checkpoint boundary.
|
||||||
var records = _wal.ReadAll();
|
var records = _wal.ReadAll();
|
||||||
var startIndex = 0;
|
var startIndex = 0;
|
||||||
for (var i = records.Count - 1; i >= 0; i--)
|
for (int i = records.Count - 1; i >= 0; i--)
|
||||||
{
|
|
||||||
if (records[i].Type == WalRecordType.Checkpoint)
|
if (records[i].Type == WalRecordType.Checkpoint)
|
||||||
{
|
{
|
||||||
startIndex = i + 1;
|
startIndex = i + 1;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Replay WAL in source order with deterministic commit application.
|
// 2. Replay WAL in source order with deterministic commit application.
|
||||||
var pendingWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
var pendingWrites = new Dictionary<ulong, List<(uint pageId, byte[] data)>>();
|
||||||
var appliedAny = false;
|
var appliedAny = false;
|
||||||
|
|
||||||
for (var i = startIndex; i < records.Count; i++)
|
for (int i = startIndex; i < records.Count; i++)
|
||||||
{
|
{
|
||||||
var record = records[i];
|
var record = records[i];
|
||||||
switch (record.Type)
|
switch (record.Type)
|
||||||
{
|
{
|
||||||
case WalRecordType.Begin:
|
case WalRecordType.Begin:
|
||||||
if (!pendingWrites.ContainsKey(record.TransactionId))
|
if (!pendingWrites.ContainsKey(record.TransactionId))
|
||||||
{
|
|
||||||
pendingWrites[record.TransactionId] = new List<(uint, byte[])>();
|
pendingWrites[record.TransactionId] = new List<(uint, byte[])>();
|
||||||
}
|
|
||||||
break;
|
break;
|
||||||
case WalRecordType.Write:
|
case WalRecordType.Write:
|
||||||
if (record.AfterImage == null)
|
if (record.AfterImage == null) break;
|
||||||
{
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!pendingWrites.TryGetValue(record.TransactionId, out var writes))
|
if (!pendingWrites.TryGetValue(record.TransactionId, out var writes))
|
||||||
{
|
{
|
||||||
@@ -228,12 +216,9 @@ public sealed partial class StorageEngine
|
|||||||
writes.Add((record.PageId, record.AfterImage));
|
writes.Add((record.PageId, record.AfterImage));
|
||||||
break;
|
break;
|
||||||
case WalRecordType.Commit:
|
case WalRecordType.Commit:
|
||||||
if (!pendingWrites.TryGetValue(record.TransactionId, out var committedWrites))
|
if (!pendingWrites.TryGetValue(record.TransactionId, out var committedWrites)) break;
|
||||||
{
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (var (pageId, data) in committedWrites)
|
foreach ((uint pageId, byte[] data) in committedWrites)
|
||||||
{
|
{
|
||||||
_pageFile.WritePage(pageId, data);
|
_pageFile.WritePage(pageId, data);
|
||||||
appliedAny = true;
|
appliedAny = true;
|
||||||
@@ -251,23 +236,17 @@ public sealed partial class StorageEngine
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 3. Flush PageFile to ensure durability.
|
// 3. Flush PageFile to ensure durability.
|
||||||
if (appliedAny)
|
if (appliedAny) _pageFile.Flush();
|
||||||
{
|
|
||||||
_pageFile.Flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Clear in-memory WAL index (redundant since we just recovered).
|
// 4. Clear in-memory WAL index (redundant since we just recovered).
|
||||||
_walIndex.Clear();
|
_walIndex.Clear();
|
||||||
|
|
||||||
// 5. Truncate WAL (all changes now in PageFile).
|
// 5. Truncate WAL (all changes now in PageFile).
|
||||||
if (_wal.GetCurrentSize() > 0)
|
if (_wal.GetCurrentSize() > 0) _wal.Truncate();
|
||||||
{
|
|
||||||
_wal.Truncate();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
finally
|
finally
|
||||||
{
|
{
|
||||||
_commitLock.Release();
|
_commitLock.Release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,30 +1,28 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Bson.Schema;
|
using ZB.MOM.WW.CBDD.Bson.Schema;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads all schemas from the schema page chain.
|
/// Reads all schemas from the schema page chain.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
||||||
/// <returns>The list of schemas in chain order.</returns>
|
/// <returns>The list of schemas in chain order.</returns>
|
||||||
public List<BsonSchema> GetSchemas(uint rootPageId)
|
public List<BsonSchema> GetSchemas(uint rootPageId)
|
||||||
{
|
{
|
||||||
var schemas = new List<BsonSchema>();
|
var schemas = new List<BsonSchema>();
|
||||||
if (rootPageId == 0) return schemas;
|
if (rootPageId == 0) return schemas;
|
||||||
|
|
||||||
var pageId = rootPageId;
|
uint pageId = rootPageId;
|
||||||
var buffer = new byte[PageSize];
|
var buffer = new byte[PageSize];
|
||||||
|
|
||||||
while (pageId != 0)
|
while (pageId != 0)
|
||||||
{
|
{
|
||||||
ReadPage(pageId, null, buffer);
|
ReadPage(pageId, null, buffer);
|
||||||
var header = PageHeader.ReadFrom(buffer);
|
var header = PageHeader.ReadFrom(buffer);
|
||||||
|
|
||||||
if (header.PageType != PageType.Schema) break;
|
if (header.PageType != PageType.Schema) break;
|
||||||
|
|
||||||
int used = PageSize - 32 - header.FreeBytes;
|
int used = PageSize - 32 - header.FreeBytes;
|
||||||
@@ -33,7 +31,7 @@ public sealed partial class StorageEngine
|
|||||||
var reader = new BsonSpanReader(buffer.AsSpan(32, used), GetKeyReverseMap());
|
var reader = new BsonSpanReader(buffer.AsSpan(32, used), GetKeyReverseMap());
|
||||||
while (reader.Remaining >= 4)
|
while (reader.Remaining >= 4)
|
||||||
{
|
{
|
||||||
var docSize = reader.PeekInt32();
|
int docSize = reader.PeekInt32();
|
||||||
if (docSize <= 0 || docSize > reader.Remaining) break;
|
if (docSize <= 0 || docSize > reader.Remaining) break;
|
||||||
|
|
||||||
var schema = BsonSchema.FromBson(ref reader);
|
var schema = BsonSchema.FromBson(ref reader);
|
||||||
@@ -47,27 +45,27 @@ public sealed partial class StorageEngine
|
|||||||
return schemas;
|
return schemas;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Appends a new schema version. Returns the root page ID (which might be new if it was 0 initially).
|
/// Appends a new schema version. Returns the root page ID (which might be new if it was 0 initially).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
/// <param name="rootPageId">The root page identifier of the schema chain.</param>
|
||||||
/// <param name="schema">The schema to append.</param>
|
/// <param name="schema">The schema to append.</param>
|
||||||
public uint AppendSchema(uint rootPageId, BsonSchema schema)
|
public uint AppendSchema(uint rootPageId, BsonSchema schema)
|
||||||
{
|
{
|
||||||
var buffer = new byte[PageSize];
|
var buffer = new byte[PageSize];
|
||||||
|
|
||||||
// Serialize schema to temporary buffer to calculate size
|
// Serialize schema to temporary buffer to calculate size
|
||||||
var tempBuffer = new byte[PageSize];
|
var tempBuffer = new byte[PageSize];
|
||||||
var tempWriter = new BsonSpanWriter(tempBuffer, GetKeyMap());
|
var tempWriter = new BsonSpanWriter(tempBuffer, GetKeyMap());
|
||||||
schema.ToBson(ref tempWriter);
|
schema.ToBson(ref tempWriter);
|
||||||
var schemaSize = tempWriter.Position;
|
int schemaSize = tempWriter.Position;
|
||||||
|
|
||||||
if (rootPageId == 0)
|
if (rootPageId == 0)
|
||||||
{
|
{
|
||||||
rootPageId = AllocatePage();
|
rootPageId = AllocatePage();
|
||||||
InitializeSchemaPage(buffer, rootPageId);
|
InitializeSchemaPage(buffer, rootPageId);
|
||||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
||||||
|
|
||||||
var header = PageHeader.ReadFrom(buffer);
|
var header = PageHeader.ReadFrom(buffer);
|
||||||
header.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
header.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
||||||
header.WriteTo(buffer);
|
header.WriteTo(buffer);
|
||||||
@@ -91,13 +89,13 @@ public sealed partial class StorageEngine
|
|||||||
// Buffer now contains the last page
|
// Buffer now contains the last page
|
||||||
var lastHeader = PageHeader.ReadFrom(buffer);
|
var lastHeader = PageHeader.ReadFrom(buffer);
|
||||||
int currentUsed = PageSize - 32 - lastHeader.FreeBytes;
|
int currentUsed = PageSize - 32 - lastHeader.FreeBytes;
|
||||||
int lastOffset = 32 + currentUsed;
|
int lastOffset = 32 + currentUsed;
|
||||||
|
|
||||||
if (lastHeader.FreeBytes >= schemaSize)
|
if (lastHeader.FreeBytes >= schemaSize)
|
||||||
{
|
{
|
||||||
// Fits in current page
|
// Fits in current page
|
||||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(lastOffset));
|
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(lastOffset));
|
||||||
|
|
||||||
lastHeader.FreeBytes -= (ushort)schemaSize;
|
lastHeader.FreeBytes -= (ushort)schemaSize;
|
||||||
lastHeader.WriteTo(buffer);
|
lastHeader.WriteTo(buffer);
|
||||||
|
|
||||||
@@ -106,14 +104,14 @@ public sealed partial class StorageEngine
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Allocate new page
|
// Allocate new page
|
||||||
var newPageId = AllocatePage();
|
uint newPageId = AllocatePage();
|
||||||
lastHeader.NextPageId = newPageId;
|
lastHeader.NextPageId = newPageId;
|
||||||
lastHeader.WriteTo(buffer);
|
lastHeader.WriteTo(buffer);
|
||||||
WritePageImmediate(lastPageId, buffer);
|
WritePageImmediate(lastPageId, buffer);
|
||||||
|
|
||||||
InitializeSchemaPage(buffer, newPageId);
|
InitializeSchemaPage(buffer, newPageId);
|
||||||
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
tempBuffer.AsSpan(0, schemaSize).CopyTo(buffer.AsSpan(32));
|
||||||
|
|
||||||
var newHeader = PageHeader.ReadFrom(buffer);
|
var newHeader = PageHeader.ReadFrom(buffer);
|
||||||
newHeader.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
newHeader.FreeBytes = (ushort)(PageSize - 32 - schemaSize);
|
||||||
newHeader.WriteTo(buffer);
|
newHeader.WriteTo(buffer);
|
||||||
@@ -145,4 +143,4 @@ public sealed partial class StorageEngine
|
|||||||
var doc = reader.RemainingBytes();
|
var doc = reader.RemainingBytes();
|
||||||
doc.CopyTo(page.Slice(32));
|
doc.CopyTo(page.Slice(32));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,194 +1,75 @@
|
|||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
public sealed partial class StorageEngine
|
public sealed partial class StorageEngine
|
||||||
{
|
{
|
||||||
#region Transaction Management
|
/// <summary>
|
||||||
|
/// Gets the number of active transactions (diagnostics).
|
||||||
|
/// </summary>
|
||||||
|
public int ActiveTransactionCount => _walCache.Count;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Begins a new transaction.
|
/// Prepares a transaction: writes all changes to WAL but doesn't commit yet.
|
||||||
|
/// Part of 2-Phase Commit protocol.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
/// <param name="transactionId">Transaction ID</param>
|
||||||
/// <returns>The started transaction.</returns>
|
/// <param name="writeSet">All writes to record in WAL</param>
|
||||||
public Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
/// <returns>True if preparation succeeded</returns>
|
||||||
|
public bool PrepareTransaction(ulong transactionId)
|
||||||
{
|
{
|
||||||
_commitLock.Wait();
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var txnId = _nextTransactionId++;
|
|
||||||
var transaction = new Transaction(txnId, this, isolationLevel);
|
|
||||||
_activeTransactions[txnId] = transaction;
|
|
||||||
return transaction;
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
_commitLock.Release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Begins a new transaction asynchronously.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
|
||||||
/// <param name="ct">The cancellation token.</param>
|
|
||||||
/// <returns>The started transaction.</returns>
|
|
||||||
public async Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted, CancellationToken ct = default)
|
|
||||||
{
|
|
||||||
await _commitLock.WaitAsync(ct);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var txnId = _nextTransactionId++;
|
|
||||||
var transaction = new Transaction(txnId, this, isolationLevel);
|
|
||||||
_activeTransactions[txnId] = transaction;
|
|
||||||
return transaction;
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
_commitLock.Release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Commits the specified transaction.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="transaction">The transaction to commit.</param>
|
|
||||||
public void CommitTransaction(Transaction transaction)
|
|
||||||
{
|
|
||||||
_commitLock.Wait();
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
_wal.WriteBeginRecord(transactionId);
|
||||||
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
|
||||||
|
|
||||||
// 1. Prepare (Write to WAL)
|
|
||||||
// In a fuller 2PC, this would be separate. Here we do it as part of commit.
|
|
||||||
if (!PrepareTransaction(transaction.TransactionId))
|
|
||||||
throw new IOException("Failed to write transaction to WAL");
|
|
||||||
|
|
||||||
// 2. Commit (Write commit record, flush, move to cache)
|
foreach (var walEntry in _walCache[transactionId])
|
||||||
// Use core commit path to avoid re-entering _commitLock.
|
_wal.WriteDataRecord(transactionId, walEntry.Key, walEntry.Value);
|
||||||
CommitTransactionCore(transaction.TransactionId);
|
|
||||||
|
|
||||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
_wal.Flush(); // Ensure WAL is persisted
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
finally
|
catch
|
||||||
{
|
{
|
||||||
_commitLock.Release();
|
// TODO: Log error?
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Commits the specified transaction asynchronously.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="transaction">The transaction to commit.</param>
|
|
||||||
/// <param name="ct">The cancellation token.</param>
|
|
||||||
public async Task CommitTransactionAsync(Transaction transaction, CancellationToken ct = default)
|
|
||||||
{
|
|
||||||
await _commitLock.WaitAsync(ct);
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
|
||||||
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
|
||||||
|
|
||||||
// 1. Prepare (Write to WAL)
|
|
||||||
if (!await PrepareTransactionAsync(transaction.TransactionId, ct))
|
|
||||||
throw new IOException("Failed to write transaction to WAL");
|
|
||||||
|
|
||||||
// 2. Commit (Write commit record, flush, move to cache)
|
|
||||||
// Use core commit path to avoid re-entering _commitLock.
|
|
||||||
await CommitTransactionCoreAsync(transaction.TransactionId, ct);
|
|
||||||
|
|
||||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
|
||||||
}
|
|
||||||
finally
|
|
||||||
{
|
|
||||||
_commitLock.Release();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Rolls back the specified transaction.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="transaction">The transaction to roll back.</param>
|
|
||||||
public void RollbackTransaction(Transaction transaction)
|
|
||||||
{
|
|
||||||
RollbackTransaction(transaction.TransactionId);
|
|
||||||
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rollback doesn't usually require async logic unless logging abort record is async,
|
|
||||||
// but for consistency we might consider it. For now, sync is fine as it's not the happy path bottleneck.
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Prepares a transaction: writes all changes to WAL but doesn't commit yet.
|
|
||||||
/// Part of 2-Phase Commit protocol.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="transactionId">Transaction ID</param>
|
|
||||||
/// <param name="writeSet">All writes to record in WAL</param>
|
|
||||||
/// <returns>True if preparation succeeded</returns>
|
|
||||||
public bool PrepareTransaction(ulong transactionId)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
_wal.WriteBeginRecord(transactionId);
|
|
||||||
|
|
||||||
foreach (var walEntry in _walCache[transactionId])
|
|
||||||
{
|
|
||||||
_wal.WriteDataRecord(transactionId, walEntry.Key, walEntry.Value);
|
|
||||||
}
|
|
||||||
|
|
||||||
_wal.Flush(); // Ensure WAL is persisted
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
// TODO: Log error?
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prepares a transaction asynchronously by writing pending changes to the WAL.
|
/// Prepares a transaction asynchronously by writing pending changes to the WAL.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">The transaction identifier.</param>
|
/// <param name="transactionId">The transaction identifier.</param>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
/// <returns><see langword="true"/> if preparation succeeds; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if preparation succeeds; otherwise, <see langword="false" />.</returns>
|
||||||
public async Task<bool> PrepareTransactionAsync(ulong transactionId, CancellationToken ct = default)
|
public async Task<bool> PrepareTransactionAsync(ulong transactionId, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
||||||
|
|
||||||
if (_walCache.TryGetValue(transactionId, out var changes))
|
if (_walCache.TryGetValue(transactionId, out var changes))
|
||||||
{
|
foreach (var walEntry in changes)
|
||||||
foreach (var walEntry in changes)
|
await _wal.WriteDataRecordAsync(transactionId, walEntry.Key, walEntry.Value, ct);
|
||||||
{
|
|
||||||
await _wal.WriteDataRecordAsync(transactionId, walEntry.Key, walEntry.Value, ct);
|
await _wal.FlushAsync(ct); // Ensure WAL is persisted
|
||||||
}
|
return true;
|
||||||
}
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
await _wal.FlushAsync(ct); // Ensure WAL is persisted
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits a transaction:
|
/// Commits a transaction:
|
||||||
/// 1. Writes all changes to WAL (for durability)
|
/// 1. Writes all changes to WAL (for durability)
|
||||||
/// 2. Writes commit record
|
/// 2. Writes commit record
|
||||||
/// 3. Flushes WAL to disk
|
/// 3. Flushes WAL to disk
|
||||||
/// 4. Moves pages from cache to WAL index (for future reads)
|
/// 4. Moves pages from cache to WAL index (for future reads)
|
||||||
/// 5. Clears WAL cache
|
/// 5. Clears WAL cache
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">Transaction to commit</param>
|
/// <param name="transactionId">Transaction to commit</param>
|
||||||
/// <param name="writeSet">All writes performed in this transaction (unused, kept for compatibility)</param>
|
/// <param name="writeSet">All writes performed in this transaction (unused, kept for compatibility)</param>
|
||||||
public void CommitTransaction(ulong transactionId)
|
public void CommitTransaction(ulong transactionId)
|
||||||
{
|
{
|
||||||
_commitLock.Wait();
|
_commitLock.Wait();
|
||||||
@@ -216,10 +97,7 @@ public sealed partial class StorageEngine
|
|||||||
// 1. Write all changes to WAL (from cache, not writeSet!)
|
// 1. Write all changes to WAL (from cache, not writeSet!)
|
||||||
_wal.WriteBeginRecord(transactionId);
|
_wal.WriteBeginRecord(transactionId);
|
||||||
|
|
||||||
foreach (var (pageId, data) in pages)
|
foreach ((uint pageId, byte[] data) in pages) _wal.WriteDataRecord(transactionId, pageId, data);
|
||||||
{
|
|
||||||
_wal.WriteDataRecord(transactionId, pageId, data);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Write commit record and flush
|
// 2. Write commit record and flush
|
||||||
_wal.WriteCommitRecord(transactionId);
|
_wal.WriteCommitRecord(transactionId);
|
||||||
@@ -227,20 +105,14 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
// 3. Move pages from cache to WAL index (for reads)
|
// 3. Move pages from cache to WAL index (for reads)
|
||||||
_walCache.TryRemove(transactionId, out _);
|
_walCache.TryRemove(transactionId, out _);
|
||||||
foreach (var kvp in pages)
|
foreach (var kvp in pages) _walIndex[kvp.Key] = kvp.Value;
|
||||||
{
|
|
||||||
_walIndex[kvp.Key] = kvp.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Auto-checkpoint if WAL grows too large
|
// Auto-checkpoint if WAL grows too large
|
||||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
if (_wal.GetCurrentSize() > MaxWalSize) CheckpointInternal();
|
||||||
{
|
|
||||||
CheckpointInternal();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits a prepared transaction asynchronously by identifier.
|
/// Commits a prepared transaction asynchronously by identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">The transaction identifier.</param>
|
/// <param name="transactionId">The transaction identifier.</param>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
@@ -271,10 +143,7 @@ public sealed partial class StorageEngine
|
|||||||
// 1. Write all changes to WAL (from cache, not writeSet!)
|
// 1. Write all changes to WAL (from cache, not writeSet!)
|
||||||
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
await _wal.WriteBeginRecordAsync(transactionId, ct);
|
||||||
|
|
||||||
foreach (var (pageId, data) in pages)
|
foreach ((uint pageId, byte[] data) in pages) await _wal.WriteDataRecordAsync(transactionId, pageId, data, ct);
|
||||||
{
|
|
||||||
await _wal.WriteDataRecordAsync(transactionId, pageId, data, ct);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Write commit record and flush
|
// 2. Write commit record and flush
|
||||||
await _wal.WriteCommitRecordAsync(transactionId, ct);
|
await _wal.WriteCommitRecordAsync(transactionId, ct);
|
||||||
@@ -282,75 +151,177 @@ public sealed partial class StorageEngine
|
|||||||
|
|
||||||
// 3. Move pages from cache to WAL index (for reads)
|
// 3. Move pages from cache to WAL index (for reads)
|
||||||
_walCache.TryRemove(transactionId, out _);
|
_walCache.TryRemove(transactionId, out _);
|
||||||
foreach (var kvp in pages)
|
foreach (var kvp in pages) _walIndex[kvp.Key] = kvp.Value;
|
||||||
{
|
|
||||||
_walIndex[kvp.Key] = kvp.Value;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Auto-checkpoint if WAL grows too large
|
// Auto-checkpoint if WAL grows too large
|
||||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
if (_wal.GetCurrentSize() > MaxWalSize)
|
||||||
{
|
|
||||||
// Checkpoint might be sync or async. For now sync inside the lock is "safe" but blocking.
|
// Checkpoint might be sync or async. For now sync inside the lock is "safe" but blocking.
|
||||||
// Ideally this should be async too.
|
// Ideally this should be async too.
|
||||||
CheckpointInternal();
|
CheckpointInternal();
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks a transaction as committed after WAL writes.
|
/// Marks a transaction as committed after WAL writes.
|
||||||
/// Used for 2PC: after Prepare() writes to WAL, this finalizes the commit.
|
/// Used for 2PC: after Prepare() writes to WAL, this finalizes the commit.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">Transaction to mark committed</param>
|
/// <param name="transactionId">Transaction to mark committed</param>
|
||||||
public void MarkTransactionCommitted(ulong transactionId)
|
public void MarkTransactionCommitted(ulong transactionId)
|
||||||
{
|
{
|
||||||
_commitLock.Wait();
|
_commitLock.Wait();
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_wal.WriteCommitRecord(transactionId);
|
_wal.WriteCommitRecord(transactionId);
|
||||||
_wal.Flush();
|
_wal.Flush();
|
||||||
|
|
||||||
// Move from cache to WAL index
|
// Move from cache to WAL index
|
||||||
if (_walCache.TryRemove(transactionId, out var pages))
|
if (_walCache.TryRemove(transactionId, out var pages))
|
||||||
{
|
foreach (var kvp in pages)
|
||||||
foreach (var kvp in pages)
|
_walIndex[kvp.Key] = kvp.Value;
|
||||||
{
|
|
||||||
_walIndex[kvp.Key] = kvp.Value;
|
// Auto-checkpoint if WAL grows too large
|
||||||
}
|
if (_wal.GetCurrentSize() > MaxWalSize) CheckpointInternal();
|
||||||
}
|
}
|
||||||
|
finally
|
||||||
// Auto-checkpoint if WAL grows too large
|
{
|
||||||
if (_wal.GetCurrentSize() > MaxWalSize)
|
_commitLock.Release();
|
||||||
{
|
}
|
||||||
CheckpointInternal();
|
}
|
||||||
}
|
|
||||||
}
|
/// <summary>
|
||||||
finally
|
/// Rolls back a transaction: discards all uncommitted changes.
|
||||||
{
|
/// </summary>
|
||||||
_commitLock.Release();
|
/// <param name="transactionId">Transaction to rollback</param>
|
||||||
}
|
public void RollbackTransaction(ulong transactionId)
|
||||||
}
|
{
|
||||||
|
_walCache.TryRemove(transactionId, out _);
|
||||||
/// <summary>
|
|
||||||
/// Rolls back a transaction: discards all uncommitted changes.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="transactionId">Transaction to rollback</param>
|
|
||||||
public void RollbackTransaction(ulong transactionId)
|
|
||||||
{
|
|
||||||
_walCache.TryRemove(transactionId, out _);
|
|
||||||
_wal.WriteAbortRecord(transactionId);
|
_wal.WriteAbortRecord(transactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes an abort record for the specified transaction.
|
/// Writes an abort record for the specified transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">The transaction identifier.</param>
|
/// <param name="transactionId">The transaction identifier.</param>
|
||||||
internal void WriteAbortRecord(ulong transactionId)
|
internal void WriteAbortRecord(ulong transactionId)
|
||||||
{
|
{
|
||||||
_wal.WriteAbortRecord(transactionId);
|
_wal.WriteAbortRecord(transactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
#region Transaction Management
|
||||||
/// Gets the number of active transactions (diagnostics).
|
|
||||||
/// </summary>
|
/// <summary>
|
||||||
public int ActiveTransactionCount => _walCache.Count;
|
/// Begins a new transaction.
|
||||||
}
|
/// </summary>
|
||||||
|
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||||
|
/// <returns>The started transaction.</returns>
|
||||||
|
public Transaction BeginTransaction(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
||||||
|
{
|
||||||
|
_commitLock.Wait();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ulong txnId = _nextTransactionId++;
|
||||||
|
var transaction = new Transaction(txnId, this, isolationLevel);
|
||||||
|
_activeTransactions[txnId] = transaction;
|
||||||
|
return transaction;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_commitLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Begins a new transaction asynchronously.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||||
|
/// <param name="ct">The cancellation token.</param>
|
||||||
|
/// <returns>The started transaction.</returns>
|
||||||
|
public async Task<Transaction> BeginTransactionAsync(IsolationLevel isolationLevel = IsolationLevel.ReadCommitted,
|
||||||
|
CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
await _commitLock.WaitAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
ulong txnId = _nextTransactionId++;
|
||||||
|
var transaction = new Transaction(txnId, this, isolationLevel);
|
||||||
|
_activeTransactions[txnId] = transaction;
|
||||||
|
return transaction;
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_commitLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Commits the specified transaction.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="transaction">The transaction to commit.</param>
|
||||||
|
public void CommitTransaction(Transaction transaction)
|
||||||
|
{
|
||||||
|
_commitLock.Wait();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
||||||
|
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
||||||
|
|
||||||
|
// 1. Prepare (Write to WAL)
|
||||||
|
// In a fuller 2PC, this would be separate. Here we do it as part of commit.
|
||||||
|
if (!PrepareTransaction(transaction.TransactionId))
|
||||||
|
throw new IOException("Failed to write transaction to WAL");
|
||||||
|
|
||||||
|
// 2. Commit (Write commit record, flush, move to cache)
|
||||||
|
// Use core commit path to avoid re-entering _commitLock.
|
||||||
|
CommitTransactionCore(transaction.TransactionId);
|
||||||
|
|
||||||
|
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_commitLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Commits the specified transaction asynchronously.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="transaction">The transaction to commit.</param>
|
||||||
|
/// <param name="ct">The cancellation token.</param>
|
||||||
|
public async Task CommitTransactionAsync(Transaction transaction, CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
await _commitLock.WaitAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (!_activeTransactions.ContainsKey(transaction.TransactionId))
|
||||||
|
throw new InvalidOperationException($"Transaction {transaction.TransactionId} is not active.");
|
||||||
|
|
||||||
|
// 1. Prepare (Write to WAL)
|
||||||
|
if (!await PrepareTransactionAsync(transaction.TransactionId, ct))
|
||||||
|
throw new IOException("Failed to write transaction to WAL");
|
||||||
|
|
||||||
|
// 2. Commit (Write commit record, flush, move to cache)
|
||||||
|
// Use core commit path to avoid re-entering _commitLock.
|
||||||
|
await CommitTransactionCoreAsync(transaction.TransactionId, ct);
|
||||||
|
|
||||||
|
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_commitLock.Release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Rolls back the specified transaction.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="transaction">The transaction to roll back.</param>
|
||||||
|
public void RollbackTransaction(Transaction transaction)
|
||||||
|
{
|
||||||
|
RollbackTransaction(transaction.TransactionId);
|
||||||
|
_activeTransactions.TryRemove(transaction.TransactionId, out _);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rollback doesn't usually require async logic unless logging abort record is async,
|
||||||
|
// but for consistency we might consider it. For now, sync is fine as it's not the happy path bottleneck.
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
}
|
||||||
@@ -1,29 +1,30 @@
|
|||||||
using System.Collections.Concurrent;
|
using System.Collections.Concurrent;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
using ZB.MOM.WW.CBDD.Core.Transactions;
|
using ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Central storage engine managing page-based storage with WAL for durability.
|
/// Central storage engine managing page-based storage with WAL for durability.
|
||||||
///
|
/// Architecture (WAL-based like SQLite/PostgreSQL):
|
||||||
/// Architecture (WAL-based like SQLite/PostgreSQL):
|
/// - PageFile: Committed baseline (persistent on disk)
|
||||||
/// - PageFile: Committed baseline (persistent on disk)
|
/// - WAL Cache: Uncommitted transaction writes (in-memory)
|
||||||
/// - WAL Cache: Uncommitted transaction writes (in-memory)
|
/// - Read: PageFile + WAL cache overlay (for Read Your Own Writes)
|
||||||
/// - Read: PageFile + WAL cache overlay (for Read Your Own Writes)
|
/// - Commit: Flush to WAL, clear cache
|
||||||
/// - Commit: Flush to WAL, clear cache
|
/// - Checkpoint: Merge WAL ? PageFile periodically
|
||||||
/// - Checkpoint: Merge WAL ? PageFile periodically
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
||||||
{
|
{
|
||||||
|
private const long MaxWalSize = 4 * 1024 * 1024; // 4MB
|
||||||
|
|
||||||
|
// Transaction Management
|
||||||
|
private readonly ConcurrentDictionary<ulong, Transaction> _activeTransactions;
|
||||||
|
|
||||||
|
// Global lock for commit/checkpoint synchronization
|
||||||
|
private readonly SemaphoreSlim _commitLock = new(1, 1);
|
||||||
private readonly PageFile _pageFile;
|
private readonly PageFile _pageFile;
|
||||||
private readonly WriteAheadLog _wal;
|
private readonly WriteAheadLog _wal;
|
||||||
private readonly CompressionOptions _compressionOptions;
|
|
||||||
private readonly CompressionService _compressionService;
|
|
||||||
private readonly CompressionTelemetry _compressionTelemetry;
|
|
||||||
private readonly StorageFormatMetadata _storageFormatMetadata;
|
|
||||||
private readonly MaintenanceOptions _maintenanceOptions;
|
|
||||||
private CDC.ChangeStreamDispatcher? _cdc;
|
|
||||||
|
|
||||||
// WAL cache: TransactionId → (PageId → PageData)
|
// WAL cache: TransactionId → (PageId → PageData)
|
||||||
// Stores uncommitted writes for "Read Your Own Writes" isolation
|
// Stores uncommitted writes for "Read Your Own Writes" isolation
|
||||||
@@ -32,18 +33,10 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
// WAL index cache: PageId → PageData (from latest committed transaction)
|
// WAL index cache: PageId → PageData (from latest committed transaction)
|
||||||
// Lazily populated on first read after commit
|
// Lazily populated on first read after commit
|
||||||
private readonly ConcurrentDictionary<uint, byte[]> _walIndex;
|
private readonly ConcurrentDictionary<uint, byte[]> _walIndex;
|
||||||
|
|
||||||
// Global lock for commit/checkpoint synchronization
|
|
||||||
private readonly SemaphoreSlim _commitLock = new(1, 1);
|
|
||||||
|
|
||||||
// Transaction Management
|
|
||||||
private readonly ConcurrentDictionary<ulong, Transaction> _activeTransactions;
|
|
||||||
private ulong _nextTransactionId;
|
private ulong _nextTransactionId;
|
||||||
|
|
||||||
private const long MaxWalSize = 4 * 1024 * 1024; // 4MB
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new instance of the <see cref="StorageEngine"/> class.
|
/// Initializes a new instance of the <see cref="StorageEngine" /> class.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="databasePath">The database file path.</param>
|
/// <param name="databasePath">The database file path.</param>
|
||||||
/// <param name="config">The page file configuration.</param>
|
/// <param name="config">The page file configuration.</param>
|
||||||
@@ -55,13 +48,13 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
CompressionOptions? compressionOptions = null,
|
CompressionOptions? compressionOptions = null,
|
||||||
MaintenanceOptions? maintenanceOptions = null)
|
MaintenanceOptions? maintenanceOptions = null)
|
||||||
{
|
{
|
||||||
_compressionOptions = CompressionOptions.Normalize(compressionOptions);
|
CompressionOptions = CompressionOptions.Normalize(compressionOptions);
|
||||||
_compressionService = new CompressionService();
|
CompressionService = new CompressionService();
|
||||||
_compressionTelemetry = new CompressionTelemetry();
|
CompressionTelemetry = new CompressionTelemetry();
|
||||||
_maintenanceOptions = maintenanceOptions ?? new MaintenanceOptions();
|
MaintenanceOptions = maintenanceOptions ?? new MaintenanceOptions();
|
||||||
|
|
||||||
// Auto-derive WAL path
|
// Auto-derive WAL path
|
||||||
var walPath = Path.ChangeExtension(databasePath, ".wal");
|
string walPath = Path.ChangeExtension(databasePath, ".wal");
|
||||||
|
|
||||||
// Initialize storage infrastructure
|
// Initialize storage infrastructure
|
||||||
_pageFile = new PageFile(databasePath, config);
|
_pageFile = new PageFile(databasePath, config);
|
||||||
@@ -72,14 +65,11 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
_walIndex = new ConcurrentDictionary<uint, byte[]>();
|
_walIndex = new ConcurrentDictionary<uint, byte[]>();
|
||||||
_activeTransactions = new ConcurrentDictionary<ulong, Transaction>();
|
_activeTransactions = new ConcurrentDictionary<ulong, Transaction>();
|
||||||
_nextTransactionId = 1;
|
_nextTransactionId = 1;
|
||||||
_storageFormatMetadata = InitializeStorageFormatMetadata();
|
StorageFormatMetadata = InitializeStorageFormatMetadata();
|
||||||
|
|
||||||
// Recover from WAL if exists (crash recovery or resume after close)
|
// Recover from WAL if exists (crash recovery or resume after close)
|
||||||
// This replays any committed transactions not yet checkpointed
|
// This replays any committed transactions not yet checkpointed
|
||||||
if (_wal.GetCurrentSize() > 0)
|
if (_wal.GetCurrentSize() > 0) Recover();
|
||||||
{
|
|
||||||
Recover();
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = ResumeCompactionIfNeeded();
|
_ = ResumeCompactionIfNeeded();
|
||||||
|
|
||||||
@@ -92,58 +82,59 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Page size for this storage engine
|
/// Compression options for this engine instance.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionOptions CompressionOptions { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compression codec service for payload roundtrip operations.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionService CompressionService { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Compression telemetry counters for this engine instance.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionTelemetry CompressionTelemetry { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets storage format metadata associated with the current database.
|
||||||
|
/// </summary>
|
||||||
|
internal StorageFormatMetadata StorageFormatMetadata { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the registered change stream dispatcher, if available.
|
||||||
|
/// </summary>
|
||||||
|
internal ChangeStreamDispatcher? Cdc { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Page size for this storage engine
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int PageSize => _pageFile.PageSize;
|
public int PageSize => _pageFile.PageSize;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Compression options for this engine instance.
|
/// Checks if a page is currently being modified by another active transaction.
|
||||||
/// </summary>
|
/// This is used to implement pessimistic locking for page allocation/selection.
|
||||||
public CompressionOptions CompressionOptions => _compressionOptions;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compression codec service for payload roundtrip operations.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionService CompressionService => _compressionService;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Compression telemetry counters for this engine instance.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionTelemetry CompressionTelemetry => _compressionTelemetry;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Returns a point-in-time snapshot of compression telemetry counters.
|
|
||||||
/// </summary>
|
|
||||||
public CompressionStats GetCompressionStats() => _compressionTelemetry.GetSnapshot();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets storage format metadata associated with the current database.
|
|
||||||
/// </summary>
|
|
||||||
internal StorageFormatMetadata StorageFormatMetadata => _storageFormatMetadata;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Checks if a page is currently being modified by another active transaction.
|
|
||||||
/// This is used to implement pessimistic locking for page allocation/selection.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="pageId">The page identifier to check.</param>
|
/// <param name="pageId">The page identifier to check.</param>
|
||||||
/// <param name="excludingTxId">The transaction identifier to exclude from the check.</param>
|
/// <param name="excludingTxId">The transaction identifier to exclude from the check.</param>
|
||||||
/// <returns><see langword="true"/> if another transaction holds the page; otherwise, <see langword="false"/>.</returns>
|
/// <returns><see langword="true" /> if another transaction holds the page; otherwise, <see langword="false" />.</returns>
|
||||||
public bool IsPageLocked(uint pageId, ulong excludingTxId)
|
public bool IsPageLocked(uint pageId, ulong excludingTxId)
|
||||||
{
|
{
|
||||||
foreach (var kvp in _walCache)
|
foreach (var kvp in _walCache)
|
||||||
{
|
{
|
||||||
var txId = kvp.Key;
|
ulong txId = kvp.Key;
|
||||||
if (txId == excludingTxId) continue;
|
if (txId == excludingTxId) continue;
|
||||||
|
|
||||||
var txnPages = kvp.Value;
|
var txnPages = kvp.Value;
|
||||||
if (txnPages.ContainsKey(pageId))
|
if (txnPages.ContainsKey(pageId))
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Disposes the storage engine and closes WAL.
|
/// Disposes the storage engine and closes WAL.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
@@ -151,13 +142,15 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
if (_activeTransactions != null)
|
if (_activeTransactions != null)
|
||||||
{
|
{
|
||||||
foreach (var txn in _activeTransactions.Values)
|
foreach (var txn in _activeTransactions.Values)
|
||||||
{
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
RollbackTransaction(txn.TransactionId);
|
RollbackTransaction(txn.TransactionId);
|
||||||
}
|
}
|
||||||
catch { /* Ignore errors during dispose */ }
|
catch
|
||||||
}
|
{
|
||||||
|
/* Ignore errors during dispose */
|
||||||
|
}
|
||||||
|
|
||||||
_activeTransactions.Clear();
|
_activeTransactions.Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,32 +161,38 @@ public sealed partial class StorageEngine : IStorageEngine, IDisposable
|
|||||||
_commitLock?.Dispose();
|
_commitLock?.Dispose();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <inheritdoc />
|
||||||
/// Registers the change stream dispatcher used for CDC notifications.
|
void IStorageEngine.RegisterCdc(ChangeStreamDispatcher cdc)
|
||||||
/// </summary>
|
|
||||||
/// <param name="cdc">The change stream dispatcher instance.</param>
|
|
||||||
internal void RegisterCdc(CDC.ChangeStreamDispatcher cdc)
|
|
||||||
{
|
{
|
||||||
_cdc = cdc;
|
RegisterCdc(cdc);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
ChangeStreamDispatcher? IStorageEngine.Cdc => Cdc;
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
CompressionOptions IStorageEngine.CompressionOptions => CompressionOptions;
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
CompressionService IStorageEngine.CompressionService => CompressionService;
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
CompressionTelemetry IStorageEngine.CompressionTelemetry => CompressionTelemetry;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns a point-in-time snapshot of compression telemetry counters.
|
||||||
|
/// </summary>
|
||||||
|
public CompressionStats GetCompressionStats()
|
||||||
|
{
|
||||||
|
return CompressionTelemetry.GetSnapshot();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the registered change stream dispatcher, if available.
|
/// Registers the change stream dispatcher used for CDC notifications.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal CDC.ChangeStreamDispatcher? Cdc => _cdc;
|
/// <param name="cdc">The change stream dispatcher instance.</param>
|
||||||
|
internal void RegisterCdc(ChangeStreamDispatcher cdc)
|
||||||
/// <inheritdoc />
|
{
|
||||||
void IStorageEngine.RegisterCdc(CDC.ChangeStreamDispatcher cdc) => RegisterCdc(cdc);
|
Cdc = cdc;
|
||||||
|
}
|
||||||
/// <inheritdoc />
|
}
|
||||||
CDC.ChangeStreamDispatcher? IStorageEngine.Cdc => _cdc;
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
CompressionOptions IStorageEngine.CompressionOptions => _compressionOptions;
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
CompressionService IStorageEngine.CompressionService => _compressionService;
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
CompressionTelemetry IStorageEngine.CompressionTelemetry => _compressionTelemetry;
|
|
||||||
}
|
|
||||||
@@ -1,40 +1,40 @@
|
|||||||
using System.Runtime.InteropServices;
|
using System.Buffers.Binary;
|
||||||
using ZB.MOM.WW.CBDD.Core.Indexing;
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
namespace ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Page for storing HNSW Vector Index nodes.
|
/// Page for storing HNSW Vector Index nodes.
|
||||||
/// Each page stores a fixed number of nodes based on vector dimensions and M.
|
/// Each page stores a fixed number of nodes based on vector dimensions and M.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public struct VectorPage
|
public struct VectorPage
|
||||||
{
|
{
|
||||||
// Layout:
|
// Layout:
|
||||||
// [PageHeader (32)]
|
// [PageHeader (32)]
|
||||||
// [Dimensions (4)]
|
// [Dimensions (4)]
|
||||||
// [MaxM (4)]
|
// [MaxM (4)]
|
||||||
// [NodeSize (4)]
|
// [NodeSize (4)]
|
||||||
// [NodeCount (4)]
|
// [NodeCount (4)]
|
||||||
// [Nodes Data (Contiguous)...]
|
// [Nodes Data (Contiguous)...]
|
||||||
|
|
||||||
private const int DimensionsOffset = 32;
|
private const int DimensionsOffset = 32;
|
||||||
private const int MaxMOffset = 36;
|
private const int MaxMOffset = 36;
|
||||||
private const int NodeSizeOffset = 40;
|
private const int NodeSizeOffset = 40;
|
||||||
private const int NodeCountOffset = 44;
|
private const int NodeCountOffset = 44;
|
||||||
private const int DataOffset = 48;
|
private const int DataOffset = 48;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Increments the node count stored in the vector page header.
|
/// Increments the node count stored in the vector page header.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
public static void IncrementNodeCount(Span<byte> page)
|
public static void IncrementNodeCount(Span<byte> page)
|
||||||
{
|
{
|
||||||
int count = GetNodeCount(page);
|
int count = GetNodeCount(page);
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), count + 1);
|
BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), count + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a vector page with header metadata and sizing information.
|
/// Initializes a vector page with header metadata and sizing information.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="pageId">The page identifier.</param>
|
/// <param name="pageId">The page identifier.</param>
|
||||||
@@ -43,54 +43,60 @@ public struct VectorPage
|
|||||||
public static void Initialize(Span<byte> page, uint pageId, int dimensions, int maxM)
|
public static void Initialize(Span<byte> page, uint pageId, int dimensions, int maxM)
|
||||||
{
|
{
|
||||||
var header = new PageHeader
|
var header = new PageHeader
|
||||||
{
|
{
|
||||||
PageId = pageId,
|
PageId = pageId,
|
||||||
PageType = PageType.Vector,
|
PageType = PageType.Vector,
|
||||||
FreeBytes = (ushort)(page.Length - DataOffset),
|
FreeBytes = (ushort)(page.Length - DataOffset),
|
||||||
NextPageId = 0,
|
NextPageId = 0,
|
||||||
TransactionId = 0
|
TransactionId = 0
|
||||||
};
|
};
|
||||||
header.WriteTo(page);
|
header.WriteTo(page);
|
||||||
|
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(DimensionsOffset), dimensions);
|
BinaryPrimitives.WriteInt32LittleEndian(page.Slice(DimensionsOffset), dimensions);
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(MaxMOffset), maxM);
|
BinaryPrimitives.WriteInt32LittleEndian(page.Slice(MaxMOffset), maxM);
|
||||||
|
|
||||||
// Node Size Calculation:
|
// Node Size Calculation:
|
||||||
// Location (6) + MaxLevel (1) + Vector (dim * 4) + Links (maxM * 10 * 6) -- estimating 10 levels for simplicity
|
// Location (6) + MaxLevel (1) + Vector (dim * 4) + Links (maxM * 10 * 6) -- estimating 10 levels for simplicity
|
||||||
// Better: Node size is variable? No, let's keep it fixed per index configuration to avoid fragmentation.
|
// Better: Node size is variable? No, let's keep it fixed per index configuration to avoid fragmentation.
|
||||||
// HNSW standard: level 0 has 2*M links, levels > 0 have M links.
|
// HNSW standard: level 0 has 2*M links, levels > 0 have M links.
|
||||||
// Max level is typically < 16. Let's reserve space for 16 levels.
|
// Max level is typically < 16. Let's reserve space for 16 levels.
|
||||||
int nodeSize = 6 + 1 + (dimensions * 4) + (maxM * (2 + 15) * 6);
|
int nodeSize = 6 + 1 + dimensions * 4 + maxM * (2 + 15) * 6;
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeSizeOffset), nodeSize);
|
BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeSizeOffset), nodeSize);
|
||||||
System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), 0);
|
BinaryPrimitives.WriteInt32LittleEndian(page.Slice(NodeCountOffset), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the number of nodes currently stored in the page.
|
/// Gets the number of nodes currently stored in the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns>The node count.</returns>
|
/// <returns>The node count.</returns>
|
||||||
public static int GetNodeCount(ReadOnlySpan<byte> page) =>
|
public static int GetNodeCount(ReadOnlySpan<byte> page)
|
||||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeCountOffset));
|
{
|
||||||
|
return BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeCountOffset));
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the configured node size for the page.
|
/// Gets the configured node size for the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns>The node size in bytes.</returns>
|
/// <returns>The node size in bytes.</returns>
|
||||||
public static int GetNodeSize(ReadOnlySpan<byte> page) =>
|
public static int GetNodeSize(ReadOnlySpan<byte> page)
|
||||||
System.Buffers.Binary.BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeSizeOffset));
|
{
|
||||||
|
return BinaryPrimitives.ReadInt32LittleEndian(page.Slice(NodeSizeOffset));
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the maximum number of nodes that can fit in the page.
|
/// Gets the maximum number of nodes that can fit in the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <returns>The maximum node count.</returns>
|
/// <returns>The maximum node count.</returns>
|
||||||
public static int GetMaxNodes(ReadOnlySpan<byte> page) =>
|
public static int GetMaxNodes(ReadOnlySpan<byte> page)
|
||||||
(page.Length - DataOffset) / GetNodeSize(page);
|
{
|
||||||
|
return (page.Length - DataOffset) / GetNodeSize(page);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Writes a node to the page at the specified index.
|
/// Writes a node to the page at the specified index.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||||
@@ -98,50 +104,52 @@ public struct VectorPage
|
|||||||
/// <param name="maxLevel">The maximum graph level for the node.</param>
|
/// <param name="maxLevel">The maximum graph level for the node.</param>
|
||||||
/// <param name="vector">The vector values to store.</param>
|
/// <param name="vector">The vector values to store.</param>
|
||||||
/// <param name="dimensions">The vector dimensionality.</param>
|
/// <param name="dimensions">The vector dimensionality.</param>
|
||||||
public static void WriteNode(Span<byte> page, int nodeIndex, DocumentLocation loc, int maxLevel, ReadOnlySpan<float> vector, int dimensions)
|
public static void WriteNode(Span<byte> page, int nodeIndex, DocumentLocation loc, int maxLevel,
|
||||||
|
ReadOnlySpan<float> vector, int dimensions)
|
||||||
{
|
{
|
||||||
int nodeSize = GetNodeSize(page);
|
int nodeSize = GetNodeSize(page);
|
||||||
int offset = DataOffset + (nodeIndex * nodeSize);
|
int offset = DataOffset + nodeIndex * nodeSize;
|
||||||
var nodeSpan = page.Slice(offset, nodeSize);
|
var nodeSpan = page.Slice(offset, nodeSize);
|
||||||
|
|
||||||
// 1. Document Location
|
// 1. Document Location
|
||||||
loc.WriteTo(nodeSpan.Slice(0, 6));
|
loc.WriteTo(nodeSpan.Slice(0, 6));
|
||||||
|
|
||||||
// 2. Max Level
|
// 2. Max Level
|
||||||
nodeSpan[6] = (byte)maxLevel;
|
nodeSpan[6] = (byte)maxLevel;
|
||||||
|
|
||||||
// 3. Vector
|
// 3. Vector
|
||||||
var vectorSpan = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, dimensions * 4));
|
var vectorSpan = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, dimensions * 4));
|
||||||
vector.CopyTo(vectorSpan);
|
vector.CopyTo(vectorSpan);
|
||||||
|
|
||||||
// 4. Links (initialize with 0/empty)
|
// 4. Links (initialize with 0/empty)
|
||||||
// Links follow the vector. Level 0: 2*M links, Level 1..15: M links.
|
// Links follow the vector. Level 0: 2*M links, Level 1..15: M links.
|
||||||
// For now, just ensure it's cleared or handled by the indexer.
|
// For now, just ensure it's cleared or handled by the indexer.
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reads node metadata and vector data from the page.
|
/// Reads node metadata and vector data from the page.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||||
/// <param name="loc">When this method returns, contains the node document location.</param>
|
/// <param name="loc">When this method returns, contains the node document location.</param>
|
||||||
/// <param name="maxLevel">When this method returns, contains the node max level.</param>
|
/// <param name="maxLevel">When this method returns, contains the node max level.</param>
|
||||||
/// <param name="vector">The destination span for vector values.</param>
|
/// <param name="vector">The destination span for vector values.</param>
|
||||||
public static void ReadNodeData(ReadOnlySpan<byte> page, int nodeIndex, out DocumentLocation loc, out int maxLevel, Span<float> vector)
|
public static void ReadNodeData(ReadOnlySpan<byte> page, int nodeIndex, out DocumentLocation loc, out int maxLevel,
|
||||||
|
Span<float> vector)
|
||||||
{
|
{
|
||||||
int nodeSize = GetNodeSize(page);
|
int nodeSize = GetNodeSize(page);
|
||||||
int offset = DataOffset + (nodeIndex * nodeSize);
|
int offset = DataOffset + nodeIndex * nodeSize;
|
||||||
var nodeSpan = page.Slice(offset, nodeSize);
|
var nodeSpan = page.Slice(offset, nodeSize);
|
||||||
|
|
||||||
loc = DocumentLocation.ReadFrom(nodeSpan.Slice(0, 6));
|
loc = DocumentLocation.ReadFrom(nodeSpan.Slice(0, 6));
|
||||||
maxLevel = nodeSpan[6];
|
maxLevel = nodeSpan[6];
|
||||||
|
|
||||||
var vectorSource = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, vector.Length * 4));
|
var vectorSource = MemoryMarshal.Cast<byte, float>(nodeSpan.Slice(7, vector.Length * 4));
|
||||||
vectorSource.CopyTo(vector);
|
vectorSource.CopyTo(vector);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the span that stores links for a node at a specific level.
|
/// Gets the span that stores links for a node at a specific level.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="page">The page buffer.</param>
|
/// <param name="page">The page buffer.</param>
|
||||||
/// <param name="nodeIndex">The zero-based node index.</param>
|
/// <param name="nodeIndex">The zero-based node index.</param>
|
||||||
@@ -152,23 +160,19 @@ public struct VectorPage
|
|||||||
public static Span<byte> GetLinksSpan(Span<byte> page, int nodeIndex, int level, int dimensions, int maxM)
|
public static Span<byte> GetLinksSpan(Span<byte> page, int nodeIndex, int level, int dimensions, int maxM)
|
||||||
{
|
{
|
||||||
int nodeSize = GetNodeSize(page);
|
int nodeSize = GetNodeSize(page);
|
||||||
int nodeOffset = DataOffset + (nodeIndex * nodeSize);
|
int nodeOffset = DataOffset + nodeIndex * nodeSize;
|
||||||
|
|
||||||
// Link offset: Location(6) + MaxLevel(1) + Vector(dim*4)
|
// Link offset: Location(6) + MaxLevel(1) + Vector(dim*4)
|
||||||
int linkBaseOffset = nodeOffset + 7 + (dimensions * 4);
|
int linkBaseOffset = nodeOffset + 7 + dimensions * 4;
|
||||||
|
|
||||||
int levelOffset;
|
int levelOffset;
|
||||||
if (level == 0)
|
if (level == 0)
|
||||||
{
|
levelOffset = 0;
|
||||||
levelOffset = 0;
|
else
|
||||||
}
|
// Level 0 has 2*M links
|
||||||
else
|
levelOffset = 2 * maxM * 6 + (level - 1) * maxM * 6;
|
||||||
{
|
|
||||||
// Level 0 has 2*M links
|
int count = level == 0 ? 2 * maxM : maxM;
|
||||||
levelOffset = (2 * maxM * 6) + ((level - 1) * maxM * 6);
|
return page.Slice(linkBaseOffset + levelOffset, count * 6);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
int count = (level == 0) ? (2 * maxM) : maxM;
|
|
||||||
return page.Slice(linkBaseOffset + levelOffset, count * 6);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,39 +1,39 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Defines checkpoint modes for WAL (Write-Ahead Log) checkpointing.
|
/// Defines checkpoint modes for WAL (Write-Ahead Log) checkpointing.
|
||||||
/// Similar to SQLite's checkpoint strategies.
|
/// Similar to SQLite's checkpoint strategies.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum CheckpointMode
|
public enum CheckpointMode
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Passive checkpoint: non-blocking, best-effort transfer from WAL to database.
|
/// Passive checkpoint: non-blocking, best-effort transfer from WAL to database.
|
||||||
/// If the checkpoint lock is busy, the operation is skipped.
|
/// If the checkpoint lock is busy, the operation is skipped.
|
||||||
/// WAL content is preserved and a checkpoint marker is appended when work is applied.
|
/// WAL content is preserved and a checkpoint marker is appended when work is applied.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Passive = 0,
|
Passive = 0,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Full checkpoint: waits for the checkpoint lock, transfers committed pages to
|
/// Full checkpoint: waits for the checkpoint lock, transfers committed pages to
|
||||||
/// the page file, and preserves WAL content by appending a checkpoint marker.
|
/// the page file, and preserves WAL content by appending a checkpoint marker.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Full = 1,
|
Full = 1,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Truncate checkpoint: same as <see cref="Full"/> but truncates WAL after
|
/// Truncate checkpoint: same as <see cref="Full" /> but truncates WAL after
|
||||||
/// successfully applying committed pages. Use this to reclaim disk space.
|
/// successfully applying committed pages. Use this to reclaim disk space.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Truncate = 2,
|
Truncate = 2,
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Restart checkpoint: same as <see cref="Truncate"/> and then reinitializes
|
/// Restart checkpoint: same as <see cref="Truncate" /> and then reinitializes
|
||||||
/// the WAL stream for a fresh writer session.
|
/// the WAL stream for a fresh writer session.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Restart = 3
|
Restart = 3
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Result of a checkpoint execution.
|
/// Result of a checkpoint execution.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="Mode">Requested checkpoint mode.</param>
|
/// <param name="Mode">Requested checkpoint mode.</param>
|
||||||
/// <param name="Executed">True when checkpoint logic ran; false when skipped (for passive mode contention).</param>
|
/// <param name="Executed">True when checkpoint logic ran; false when skipped (for passive mode contention).</param>
|
||||||
@@ -49,4 +49,4 @@ public readonly record struct CheckpointResult(
|
|||||||
long WalBytesBefore,
|
long WalBytesBefore,
|
||||||
long WalBytesAfter,
|
long WalBytesAfter,
|
||||||
bool Truncated,
|
bool Truncated,
|
||||||
bool Restarted);
|
bool Restarted);
|
||||||
@@ -1,66 +1,62 @@
|
|||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Public interface for database transactions.
|
/// Public interface for database transactions.
|
||||||
/// Allows user-controlled transaction boundaries for batch operations.
|
/// Allows user-controlled transaction boundaries for batch operations.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <example>
|
/// <example>
|
||||||
/// using (var txn = collection.BeginTransaction())
|
/// using (var txn = collection.BeginTransaction())
|
||||||
/// {
|
/// {
|
||||||
/// collection.Insert(entity1, txn);
|
/// collection.Insert(entity1, txn);
|
||||||
/// collection.Insert(entity2, txn);
|
/// collection.Insert(entity2, txn);
|
||||||
/// txn.Commit();
|
/// txn.Commit();
|
||||||
/// }
|
/// }
|
||||||
/// </example>
|
/// </example>
|
||||||
public interface ITransaction : IDisposable
|
public interface ITransaction : IDisposable
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Unique transaction identifier
|
/// Unique transaction identifier
|
||||||
/// </summary>
|
/// </summary>
|
||||||
ulong TransactionId { get; }
|
ulong TransactionId { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Current state of the transaction
|
/// Current state of the transaction
|
||||||
/// </summary>
|
/// </summary>
|
||||||
TransactionState State { get; }
|
TransactionState State { get; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits the transaction, making all changes permanent.
|
/// Commits the transaction, making all changes permanent.
|
||||||
/// Must be called before Dispose() to persist changes.
|
/// Must be called before Dispose() to persist changes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
void Commit();
|
void Commit();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Asynchronously commits the transaction, making all changes permanent.
|
/// Asynchronously commits the transaction, making all changes permanent.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="ct">The cancellation token.</param>
|
/// <param name="ct">The cancellation token.</param>
|
||||||
Task CommitAsync(CancellationToken ct = default);
|
Task CommitAsync(CancellationToken ct = default);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Rolls back the transaction, discarding all changes.
|
/// Rolls back the transaction, discarding all changes.
|
||||||
/// Called automatically on Dispose() if Commit() was not called.
|
/// Called automatically on Dispose() if Commit() was not called.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
void Rollback();
|
void Rollback();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a write operation to the current batch or transaction.
|
/// Adds a write operation to the current batch or transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="operation">The write operation to add. Cannot be null.</param>
|
/// <param name="operation">The write operation to add. Cannot be null.</param>
|
||||||
void AddWrite(WriteOperation operation);
|
void AddWrite(WriteOperation operation);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prepares the object for use by performing any necessary initialization or setup.
|
/// Prepares the object for use by performing any necessary initialization or setup.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>true if the preparation was successful; otherwise, false.</returns>
|
/// <returns>true if the preparation was successful; otherwise, false.</returns>
|
||||||
bool Prepare();
|
bool Prepare();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Event triggered when the transaction acts rollback.
|
/// Event triggered when the transaction acts rollback.
|
||||||
/// Useful for restoring in-memory state (like ID maps).
|
/// Useful for restoring in-memory state (like ID maps).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
event Action? OnRollback;
|
event Action? OnRollback;
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,32 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Defines a contract for managing and providing access to the current transaction context.
|
/// Defines a contract for managing and providing access to the current transaction context.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>Implementations of this interface are responsible for tracking the current transaction and starting a
|
/// <remarks>
|
||||||
/// new one if none exists. This is typically used in scenarios where transactional consistency is required across
|
/// Implementations of this interface are responsible for tracking the current transaction and starting a
|
||||||
/// multiple operations.</remarks>
|
/// new one if none exists. This is typically used in scenarios where transactional consistency is required across
|
||||||
|
/// multiple operations.
|
||||||
|
/// </remarks>
|
||||||
public interface ITransactionHolder
|
public interface ITransactionHolder
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current transaction if one exists; otherwise, starts a new transaction.
|
/// Gets the current transaction if one exists; otherwise, starts a new transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>Use this method to ensure that a transaction context is available for the current operation.
|
/// <remarks>
|
||||||
/// If a transaction is already in progress, it is returned; otherwise, a new transaction is started and returned.
|
/// Use this method to ensure that a transaction context is available for the current operation.
|
||||||
/// The caller is responsible for managing the transaction's lifetime as appropriate.</remarks>
|
/// If a transaction is already in progress, it is returned; otherwise, a new transaction is started and returned.
|
||||||
/// <returns>An <see cref="ITransaction"/> representing the current transaction, or a new transaction if none is active.</returns>
|
/// The caller is responsible for managing the transaction's lifetime as appropriate.
|
||||||
|
/// </remarks>
|
||||||
|
/// <returns>An <see cref="ITransaction" /> representing the current transaction, or a new transaction if none is active.</returns>
|
||||||
ITransaction GetCurrentTransactionOrStart();
|
ITransaction GetCurrentTransactionOrStart();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the current transaction if one exists; otherwise, starts a new transaction asynchronously.
|
/// Gets the current transaction if one exists; otherwise, starts a new transaction asynchronously.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>A task that represents the asynchronous operation. The task result contains an <see cref="ITransaction"/>
|
/// <returns>
|
||||||
/// representing the current or newly started transaction.</returns>
|
/// A task that represents the asynchronous operation. The task result contains an <see cref="ITransaction" />
|
||||||
|
/// representing the current or newly started transaction.
|
||||||
|
/// </returns>
|
||||||
Task<ITransaction> GetCurrentTransactionOrStartAsync();
|
Task<ITransaction> GetCurrentTransactionOrStartAsync();
|
||||||
}
|
}
|
||||||
@@ -1,257 +1,241 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
|
using ZB.MOM.WW.CBDD.Core.CDC;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
using System;
|
|
||||||
using System.Threading;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a transaction with ACID properties.
|
/// Represents a transaction with ACID properties.
|
||||||
/// Uses MVCC (Multi-Version Concurrency Control) for isolation.
|
/// Uses MVCC (Multi-Version Concurrency Control) for isolation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public sealed class Transaction : ITransaction
|
public sealed class Transaction : ITransaction
|
||||||
{
|
{
|
||||||
private readonly ulong _transactionId;
|
private readonly List<InternalChangeEvent> _pendingChanges = new();
|
||||||
private readonly IsolationLevel _isolationLevel;
|
|
||||||
private readonly DateTime _startTime;
|
|
||||||
private readonly StorageEngine _storage;
|
private readonly StorageEngine _storage;
|
||||||
private readonly List<CDC.InternalChangeEvent> _pendingChanges = new();
|
|
||||||
private TransactionState _state;
|
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new transaction.
|
/// Initializes a new transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="transactionId">The unique transaction identifier.</param>
|
/// <param name="transactionId">The unique transaction identifier.</param>
|
||||||
/// <param name="storage">The storage engine used by this transaction.</param>
|
/// <param name="storage">The storage engine used by this transaction.</param>
|
||||||
/// <param name="isolationLevel">The transaction isolation level.</param>
|
/// <param name="isolationLevel">The transaction isolation level.</param>
|
||||||
public Transaction(ulong transactionId,
|
public Transaction(ulong transactionId,
|
||||||
StorageEngine storage,
|
StorageEngine storage,
|
||||||
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
IsolationLevel isolationLevel = IsolationLevel.ReadCommitted)
|
||||||
{
|
{
|
||||||
_transactionId = transactionId;
|
TransactionId = transactionId;
|
||||||
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
|
||||||
_isolationLevel = isolationLevel;
|
IsolationLevel = isolationLevel;
|
||||||
_startTime = DateTime.UtcNow;
|
StartTime = DateTime.UtcNow;
|
||||||
_state = TransactionState.Active;
|
State = TransactionState.Active;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a pending CDC change to be published after commit.
|
/// Gets the configured transaction isolation level.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="change">The change event to buffer.</param>
|
public IsolationLevel IsolationLevel { get; }
|
||||||
internal void AddChange(CDC.InternalChangeEvent change)
|
|
||||||
{
|
|
||||||
_pendingChanges.Add(change);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the unique transaction identifier.
|
|
||||||
/// </summary>
|
|
||||||
public ulong TransactionId => _transactionId;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the current transaction state.
|
|
||||||
/// </summary>
|
|
||||||
public TransactionState State => _state;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the configured transaction isolation level.
|
|
||||||
/// </summary>
|
|
||||||
public IsolationLevel IsolationLevel => _isolationLevel;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the UTC start time of the transaction.
|
|
||||||
/// </summary>
|
|
||||||
public DateTime StartTime => _startTime;
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Adds a write operation to the transaction's write set.
|
/// Gets the UTC start time of the transaction.
|
||||||
/// NOTE: Makes a defensive copy of the data to ensure memory safety.
|
/// </summary>
|
||||||
/// This allocation is necessary because the caller may return the buffer to a pool.
|
public DateTime StartTime { get; }
|
||||||
/// </summary>
|
|
||||||
/// <param name="operation">The write operation to add.</param>
|
/// <summary>
|
||||||
public void AddWrite(WriteOperation operation)
|
/// Gets the unique transaction identifier.
|
||||||
|
/// </summary>
|
||||||
|
public ulong TransactionId { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the current transaction state.
|
||||||
|
/// </summary>
|
||||||
|
public TransactionState State { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Adds a write operation to the transaction's write set.
|
||||||
|
/// NOTE: Makes a defensive copy of the data to ensure memory safety.
|
||||||
|
/// This allocation is necessary because the caller may return the buffer to a pool.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="operation">The write operation to add.</param>
|
||||||
|
public void AddWrite(WriteOperation operation)
|
||||||
{
|
{
|
||||||
if (_state != TransactionState.Active)
|
if (State != TransactionState.Active)
|
||||||
throw new InvalidOperationException($"Cannot add writes to transaction in state {_state}");
|
throw new InvalidOperationException($"Cannot add writes to transaction in state {State}");
|
||||||
|
|
||||||
// Defensive copy: necessary to prevent use-after-return if caller uses pooled buffers
|
// Defensive copy: necessary to prevent use-after-return if caller uses pooled buffers
|
||||||
byte[] ownedCopy = operation.NewValue.ToArray();
|
byte[] ownedCopy = operation.NewValue.ToArray();
|
||||||
// StorageEngine gestisce tutte le scritture transazionali
|
// StorageEngine gestisce tutte le scritture transazionali
|
||||||
_storage.WritePage(operation.PageId, _transactionId, ownedCopy);
|
_storage.WritePage(operation.PageId, TransactionId, ownedCopy);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prepares the transaction for commit (2PC first phase)
|
/// Prepares the transaction for commit (2PC first phase)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public bool Prepare()
|
public bool Prepare()
|
||||||
{
|
{
|
||||||
if (_state != TransactionState.Active)
|
if (State != TransactionState.Active)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
_state = TransactionState.Preparing;
|
State = TransactionState.Preparing;
|
||||||
|
|
||||||
// StorageEngine handles WAL writes
|
// StorageEngine handles WAL writes
|
||||||
return _storage.PrepareTransaction(_transactionId);
|
return _storage.PrepareTransaction(TransactionId);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Commits the transaction.
|
/// Commits the transaction.
|
||||||
/// Writes to WAL for durability and moves data to committed buffer.
|
/// Writes to WAL for durability and moves data to committed buffer.
|
||||||
/// Pages remain in memory until CheckpointManager writes them to disk.
|
/// Pages remain in memory until CheckpointManager writes them to disk.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Commit()
|
public void Commit()
|
||||||
{
|
{
|
||||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
if (State != TransactionState.Preparing && State != TransactionState.Active)
|
||||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
throw new InvalidOperationException($"Cannot commit transaction in state {State}");
|
||||||
|
|
||||||
// StorageEngine handles WAL writes and buffer management
|
|
||||||
_storage.CommitTransaction(_transactionId);
|
|
||||||
|
|
||||||
_state = TransactionState.Committed;
|
// StorageEngine handles WAL writes and buffer management
|
||||||
|
_storage.CommitTransaction(TransactionId);
|
||||||
|
|
||||||
|
State = TransactionState.Committed;
|
||||||
|
|
||||||
// Publish CDC events after successful commit
|
// Publish CDC events after successful commit
|
||||||
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
||||||
{
|
|
||||||
foreach (var change in _pendingChanges)
|
foreach (var change in _pendingChanges)
|
||||||
{
|
|
||||||
_storage.Cdc.Publish(change);
|
_storage.Cdc.Publish(change);
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Asynchronously commits the transaction.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="ct">A cancellation token.</param>
|
|
||||||
public async Task CommitAsync(CancellationToken ct = default)
|
|
||||||
{
|
|
||||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
|
||||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
|
||||||
|
|
||||||
// StorageEngine handles WAL writes and buffer management
|
|
||||||
await _storage.CommitTransactionAsync(_transactionId, ct);
|
|
||||||
|
|
||||||
_state = TransactionState.Committed;
|
|
||||||
|
|
||||||
// Publish CDC events after successful commit
|
|
||||||
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
|
||||||
{
|
|
||||||
foreach (var change in _pendingChanges)
|
|
||||||
{
|
|
||||||
_storage.Cdc.Publish(change);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks the transaction as committed without writing to PageFile.
|
/// Asynchronously commits the transaction.
|
||||||
/// Used by TransactionManager with lazy checkpointing.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
internal void MarkCommitted()
|
/// <param name="ct">A cancellation token.</param>
|
||||||
|
public async Task CommitAsync(CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
if (_state != TransactionState.Preparing && _state != TransactionState.Active)
|
if (State != TransactionState.Preparing && State != TransactionState.Active)
|
||||||
throw new InvalidOperationException($"Cannot commit transaction in state {_state}");
|
throw new InvalidOperationException($"Cannot commit transaction in state {State}");
|
||||||
|
|
||||||
// StorageEngine marks transaction as committed and moves to committed buffer
|
// StorageEngine handles WAL writes and buffer management
|
||||||
_storage.MarkTransactionCommitted(_transactionId);
|
await _storage.CommitTransactionAsync(TransactionId, ct);
|
||||||
|
|
||||||
_state = TransactionState.Committed;
|
State = TransactionState.Committed;
|
||||||
|
|
||||||
|
// Publish CDC events after successful commit
|
||||||
|
if (_pendingChanges.Count > 0 && _storage.Cdc != null)
|
||||||
|
foreach (var change in _pendingChanges)
|
||||||
|
_storage.Cdc.Publish(change);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Rolls back the transaction (discards all writes)
|
/// Rolls back the transaction (discards all writes)
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public event Action? OnRollback;
|
public event Action? OnRollback;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Rolls back the transaction and discards pending writes.
|
/// Rolls back the transaction and discards pending writes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Rollback()
|
public void Rollback()
|
||||||
{
|
{
|
||||||
if (_state == TransactionState.Committed)
|
if (State == TransactionState.Committed)
|
||||||
throw new InvalidOperationException("Cannot rollback committed transaction");
|
throw new InvalidOperationException("Cannot rollback committed transaction");
|
||||||
|
|
||||||
_pendingChanges.Clear();
|
_pendingChanges.Clear();
|
||||||
_storage.RollbackTransaction(_transactionId);
|
_storage.RollbackTransaction(TransactionId);
|
||||||
_state = TransactionState.Aborted;
|
State = TransactionState.Aborted;
|
||||||
|
|
||||||
OnRollback?.Invoke();
|
OnRollback?.Invoke();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Releases transaction resources and rolls back if still active.
|
/// Releases transaction resources and rolls back if still active.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
if (_disposed)
|
if (_disposed)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
if (_state == TransactionState.Active || _state == TransactionState.Preparing)
|
if (State == TransactionState.Active || State == TransactionState.Preparing)
|
||||||
{
|
|
||||||
// Auto-rollback if not committed
|
// Auto-rollback if not committed
|
||||||
Rollback();
|
Rollback();
|
||||||
}
|
|
||||||
|
|
||||||
_disposed = true;
|
_disposed = true;
|
||||||
GC.SuppressFinalize(this);
|
GC.SuppressFinalize(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Adds a pending CDC change to be published after commit.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="change">The change event to buffer.</param>
|
||||||
|
internal void AddChange(InternalChangeEvent change)
|
||||||
|
{
|
||||||
|
_pendingChanges.Add(change);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Marks the transaction as committed without writing to PageFile.
|
||||||
|
/// Used by TransactionManager with lazy checkpointing.
|
||||||
|
/// </summary>
|
||||||
|
internal void MarkCommitted()
|
||||||
|
{
|
||||||
|
if (State != TransactionState.Preparing && State != TransactionState.Active)
|
||||||
|
throw new InvalidOperationException($"Cannot commit transaction in state {State}");
|
||||||
|
|
||||||
|
// StorageEngine marks transaction as committed and moves to committed buffer
|
||||||
|
_storage.MarkTransactionCommitted(TransactionId);
|
||||||
|
|
||||||
|
State = TransactionState.Committed;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Represents a write operation in a transaction.
|
/// Represents a write operation in a transaction.
|
||||||
/// Optimized to avoid allocations by using ReadOnlyMemory instead of byte[].
|
/// Optimized to avoid allocations by using ReadOnlyMemory instead of byte[].
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public struct WriteOperation
|
public struct WriteOperation
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the identifier of the affected document.
|
/// Gets or sets the identifier of the affected document.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ObjectId DocumentId { get; set; }
|
public ObjectId DocumentId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the new serialized value.
|
/// Gets or sets the new serialized value.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ReadOnlyMemory<byte> NewValue { get; set; }
|
public ReadOnlyMemory<byte> NewValue { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the target page identifier.
|
/// Gets or sets the target page identifier.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public uint PageId { get; set; }
|
public uint PageId { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the operation type.
|
/// Gets or sets the operation type.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public OperationType Type { get; set; }
|
public OperationType Type { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new write operation.
|
/// Initializes a new write operation.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="documentId">The identifier of the affected document.</param>
|
/// <param name="documentId">The identifier of the affected document.</param>
|
||||||
/// <param name="newValue">The new serialized value.</param>
|
/// <param name="newValue">The new serialized value.</param>
|
||||||
/// <param name="pageId">The target page identifier.</param>
|
/// <param name="pageId">The target page identifier.</param>
|
||||||
/// <param name="type">The operation type.</param>
|
/// <param name="type">The operation type.</param>
|
||||||
public WriteOperation(ObjectId documentId, ReadOnlyMemory<byte> newValue, uint pageId, OperationType type)
|
public WriteOperation(ObjectId documentId, ReadOnlyMemory<byte> newValue, uint pageId, OperationType type)
|
||||||
{
|
{
|
||||||
DocumentId = documentId;
|
DocumentId = documentId;
|
||||||
NewValue = newValue;
|
NewValue = newValue;
|
||||||
PageId = pageId;
|
PageId = pageId;
|
||||||
Type = type;
|
Type = type;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Backward compatibility constructor
|
// Backward compatibility constructor
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Initializes a new write operation from a byte array payload.
|
/// Initializes a new write operation from a byte array payload.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="documentId">The identifier of the affected document.</param>
|
/// <param name="documentId">The identifier of the affected document.</param>
|
||||||
/// <param name="newValue">The new serialized value.</param>
|
/// <param name="newValue">The new serialized value.</param>
|
||||||
/// <param name="pageId">The target page identifier.</param>
|
/// <param name="pageId">The target page identifier.</param>
|
||||||
/// <param name="type">The operation type.</param>
|
/// <param name="type">The operation type.</param>
|
||||||
public WriteOperation(ObjectId documentId, byte[] newValue, uint pageId, OperationType type)
|
public WriteOperation(ObjectId documentId, byte[] newValue, uint pageId, OperationType type)
|
||||||
{
|
{
|
||||||
DocumentId = documentId;
|
DocumentId = documentId;
|
||||||
NewValue = newValue;
|
NewValue = newValue;
|
||||||
PageId = pageId;
|
PageId = pageId;
|
||||||
Type = type;
|
Type = type;
|
||||||
@@ -259,7 +243,7 @@ public struct WriteOperation
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Type of write operation
|
/// Type of write operation
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum OperationType : byte
|
public enum OperationType : byte
|
||||||
{
|
{
|
||||||
@@ -267,4 +251,4 @@ public enum OperationType : byte
|
|||||||
Update = 2,
|
Update = 2,
|
||||||
Delete = 3,
|
Delete = 3,
|
||||||
AllocatePage = 4
|
AllocatePage = 4
|
||||||
}
|
}
|
||||||
@@ -1,37 +1,37 @@
|
|||||||
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
namespace ZB.MOM.WW.CBDD.Core.Transactions;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Transaction states
|
/// Transaction states
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum TransactionState : byte
|
public enum TransactionState : byte
|
||||||
{
|
{
|
||||||
/// <summary>Transaction is active and can accept operations</summary>
|
/// <summary>Transaction is active and can accept operations</summary>
|
||||||
Active = 1,
|
Active = 1,
|
||||||
|
|
||||||
/// <summary>Transaction is preparing to commit</summary>
|
/// <summary>Transaction is preparing to commit</summary>
|
||||||
Preparing = 2,
|
Preparing = 2,
|
||||||
|
|
||||||
/// <summary>Transaction committed successfully</summary>
|
/// <summary>Transaction committed successfully</summary>
|
||||||
Committed = 3,
|
Committed = 3,
|
||||||
|
|
||||||
/// <summary>Transaction was rolled back</summary>
|
/// <summary>Transaction was rolled back</summary>
|
||||||
Aborted = 4
|
Aborted = 4
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Transaction isolation levels
|
/// Transaction isolation levels
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public enum IsolationLevel : byte
|
public enum IsolationLevel : byte
|
||||||
{
|
{
|
||||||
/// <summary>Read uncommitted data</summary>
|
/// <summary>Read uncommitted data</summary>
|
||||||
ReadUncommitted = 0,
|
ReadUncommitted = 0,
|
||||||
|
|
||||||
/// <summary>Read only committed data (default)</summary>
|
/// <summary>Read only committed data (default)</summary>
|
||||||
ReadCommitted = 1,
|
ReadCommitted = 1,
|
||||||
|
|
||||||
/// <summary>Repeatable reads</summary>
|
/// <summary>Repeatable reads</summary>
|
||||||
RepeatableRead = 2,
|
RepeatableRead = 2,
|
||||||
|
|
||||||
/// <summary>Serializable (full isolation)</summary>
|
/// <summary>Serializable (full isolation)</summary>
|
||||||
Serializable = 3
|
Serializable = 3
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,38 +1,38 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<AssemblyName>ZB.MOM.WW.CBDD.Core</AssemblyName>
|
<AssemblyName>ZB.MOM.WW.CBDD.Core</AssemblyName>
|
||||||
<RootNamespace>ZB.MOM.WW.CBDD.Core</RootNamespace>
|
<RootNamespace>ZB.MOM.WW.CBDD.Core</RootNamespace>
|
||||||
<LangVersion>latest</LangVersion>
|
<LangVersion>latest</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||||
|
|
||||||
<PackageId>ZB.MOM.WW.CBDD.Core</PackageId>
|
|
||||||
<Version>1.3.1</Version>
|
|
||||||
<Authors>CBDD Team</Authors>
|
|
||||||
<Description>High-Performance BSON Database Engine Core Library for .NET 10</Description>
|
|
||||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
|
||||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
|
||||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
|
||||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
<PackageId>ZB.MOM.WW.CBDD.Core</PackageId>
|
||||||
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj" />
|
<Version>1.3.1</Version>
|
||||||
</ItemGroup>
|
<Authors>CBDD Team</Authors>
|
||||||
|
<Description>High-Performance BSON Database Engine Core Library for .NET 10</Description>
|
||||||
|
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||||
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
|
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||||
|
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||||
|
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj"/>
|
||||||
<_Parameter1>ZB.MOM.WW.CBDD.Tests</_Parameter1>
|
</ItemGroup>
|
||||||
</AssemblyAttribute>
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
<AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
|
||||||
</ItemGroup>
|
<_Parameter1>ZB.MOM.WW.CBDD.Tests</_Parameter1>
|
||||||
|
</AssemblyAttribute>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="..\..\README.md" Pack="true" PackagePath="\"/>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,261 +1,257 @@
|
|||||||
using System;
|
using System.Collections.Generic;
|
||||||
using System.Collections.Generic;
|
using System.Linq;
|
||||||
using System.Linq;
|
using Microsoft.CodeAnalysis;
|
||||||
using Microsoft.CodeAnalysis;
|
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||||
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||||
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.SourceGenerators;
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators
|
|
||||||
{
|
public static class EntityAnalyzer
|
||||||
public static class EntityAnalyzer
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Analyzes an entity symbol and builds source-generation metadata.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="entityType">The entity type symbol to analyze.</param>
|
||||||
|
/// <param name="semanticModel">The semantic model for symbol and syntax analysis.</param>
|
||||||
|
/// <returns>The analyzed entity metadata.</returns>
|
||||||
|
public static EntityInfo Analyze(INamedTypeSymbol entityType, SemanticModel semanticModel)
|
||||||
{
|
{
|
||||||
/// <summary>
|
var entityInfo = new EntityInfo
|
||||||
/// Analyzes an entity symbol and builds source-generation metadata.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="entityType">The entity type symbol to analyze.</param>
|
|
||||||
/// <param name="semanticModel">The semantic model for symbol and syntax analysis.</param>
|
|
||||||
/// <returns>The analyzed entity metadata.</returns>
|
|
||||||
public static EntityInfo Analyze(INamedTypeSymbol entityType, SemanticModel semanticModel)
|
|
||||||
{
|
{
|
||||||
var entityInfo = new EntityInfo
|
Name = entityType.Name,
|
||||||
{
|
Namespace = entityType.ContainingNamespace.ToDisplayString(),
|
||||||
Name = entityType.Name,
|
FullTypeName = SyntaxHelper.GetFullName(entityType),
|
||||||
Namespace = entityType.ContainingNamespace.ToDisplayString(),
|
CollectionName = entityType.Name.ToLowerInvariant() + "s"
|
||||||
FullTypeName = SyntaxHelper.GetFullName(entityType),
|
};
|
||||||
CollectionName = entityType.Name.ToLowerInvariant() + "s"
|
|
||||||
};
|
|
||||||
|
|
||||||
var tableAttr = AttributeHelper.GetAttribute(entityType, "Table");
|
|
||||||
if (tableAttr != null)
|
|
||||||
{
|
|
||||||
var tableName = tableAttr.ConstructorArguments.Length > 0 ? tableAttr.ConstructorArguments[0].Value?.ToString() : null;
|
|
||||||
var schema = AttributeHelper.GetNamedArgumentValue(tableAttr, "Schema");
|
|
||||||
|
|
||||||
var collectionName = !string.IsNullOrEmpty(tableName) ? tableName! : entityInfo.Name;
|
var tableAttr = AttributeHelper.GetAttribute(entityType, "Table");
|
||||||
if (!string.IsNullOrEmpty(schema))
|
if (tableAttr != null)
|
||||||
{
|
{
|
||||||
collectionName = $"{schema}.{collectionName}";
|
string? tableName = tableAttr.ConstructorArguments.Length > 0
|
||||||
}
|
? tableAttr.ConstructorArguments[0].Value?.ToString()
|
||||||
entityInfo.CollectionName = collectionName;
|
: null;
|
||||||
}
|
string? schema = AttributeHelper.GetNamedArgumentValue(tableAttr, "Schema");
|
||||||
|
|
||||||
// Analyze properties of the root entity
|
string collectionName = !string.IsNullOrEmpty(tableName) ? tableName! : entityInfo.Name;
|
||||||
AnalyzeProperties(entityType, entityInfo.Properties);
|
if (!string.IsNullOrEmpty(schema)) collectionName = $"{schema}.{collectionName}";
|
||||||
|
entityInfo.CollectionName = collectionName;
|
||||||
// Check if entity needs reflection-based deserialization
|
}
|
||||||
// Include properties with private setters or init-only setters (which can't be set outside initializers)
|
|
||||||
entityInfo.HasPrivateSetters = entityInfo.Properties.Any(p => (!p.HasPublicSetter && p.HasAnySetter) || p.HasInitOnlySetter);
|
|
||||||
|
|
||||||
// Check if entity has public parameterless constructor
|
// Analyze properties of the root entity
|
||||||
var hasPublicParameterlessConstructor = entityType.Constructors
|
AnalyzeProperties(entityType, entityInfo.Properties);
|
||||||
.Any(c => c.DeclaredAccessibility == Accessibility.Public && c.Parameters.Length == 0);
|
|
||||||
entityInfo.HasPrivateOrNoConstructor = !hasPublicParameterlessConstructor;
|
|
||||||
|
|
||||||
// Analyze nested types recursively
|
|
||||||
// We use a dictionary for nested types to ensure uniqueness by name
|
|
||||||
var analyzedTypes = new HashSet<string>();
|
|
||||||
AnalyzeNestedTypesRecursive(entityInfo.Properties, entityInfo.NestedTypes, semanticModel, analyzedTypes, 1, 3);
|
|
||||||
|
|
||||||
// Determine ID property
|
// Check if entity needs reflection-based deserialization
|
||||||
// entityInfo.IdProperty is computed from Properties.FirstOrDefault(p => p.IsKey)
|
// Include properties with private setters or init-only setters (which can't be set outside initializers)
|
||||||
|
entityInfo.HasPrivateSetters =
|
||||||
|
entityInfo.Properties.Any(p => (!p.HasPublicSetter && p.HasAnySetter) || p.HasInitOnlySetter);
|
||||||
|
|
||||||
if (entityInfo.IdProperty == null)
|
// Check if entity has public parameterless constructor
|
||||||
{
|
bool hasPublicParameterlessConstructor = entityType.Constructors
|
||||||
// Fallback to convention: property named "Id"
|
.Any(c => c.DeclaredAccessibility == Accessibility.Public && c.Parameters.Length == 0);
|
||||||
var idProp = entityInfo.Properties.FirstOrDefault(p => p.Name == "Id");
|
entityInfo.HasPrivateOrNoConstructor = !hasPublicParameterlessConstructor;
|
||||||
if (idProp != null)
|
|
||||||
{
|
|
||||||
idProp.IsKey = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for AutoId (int/long keys)
|
// Analyze nested types recursively
|
||||||
if (entityInfo.IdProperty != null)
|
// We use a dictionary for nested types to ensure uniqueness by name
|
||||||
{
|
var analyzedTypes = new HashSet<string>();
|
||||||
var idType = entityInfo.IdProperty.TypeName.TrimEnd('?');
|
AnalyzeNestedTypesRecursive(entityInfo.Properties, entityInfo.NestedTypes, semanticModel, analyzedTypes, 1, 3);
|
||||||
if (idType == "int" || idType == "Int32" || idType == "long" || idType == "Int64")
|
|
||||||
{
|
|
||||||
entityInfo.AutoId = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return entityInfo;
|
// Determine ID property
|
||||||
}
|
// entityInfo.IdProperty is computed from Properties.FirstOrDefault(p => p.IsKey)
|
||||||
|
|
||||||
private static void AnalyzeProperties(INamedTypeSymbol typeSymbol, List<PropertyInfo> properties)
|
|
||||||
{
|
|
||||||
// Collect properties from the entire inheritance hierarchy
|
|
||||||
var seenProperties = new HashSet<string>();
|
|
||||||
var currentType = typeSymbol;
|
|
||||||
|
|
||||||
while (currentType != null && currentType.SpecialType != SpecialType.System_Object)
|
if (entityInfo.IdProperty == null)
|
||||||
{
|
{
|
||||||
var sourceProps = currentType.GetMembers()
|
// Fallback to convention: property named "Id"
|
||||||
.OfType<IPropertySymbol>()
|
var idProp = entityInfo.Properties.FirstOrDefault(p => p.Name == "Id");
|
||||||
.Where(p => p.DeclaredAccessibility == Accessibility.Public && !p.IsStatic);
|
if (idProp != null) idProp.IsKey = true;
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var prop in sourceProps)
|
// Check for AutoId (int/long keys)
|
||||||
{
|
if (entityInfo.IdProperty != null)
|
||||||
// Skip if already seen (overridden property in derived class takes precedence)
|
{
|
||||||
if (!seenProperties.Add(prop.Name))
|
string idType = entityInfo.IdProperty.TypeName.TrimEnd('?');
|
||||||
continue;
|
if (idType == "int" || idType == "Int32" || idType == "long" || idType == "Int64") entityInfo.AutoId = true;
|
||||||
|
}
|
||||||
|
|
||||||
if (AttributeHelper.ShouldIgnore(prop))
|
return entityInfo;
|
||||||
continue;
|
}
|
||||||
|
|
||||||
// Skip computed getter-only properties (no setter, no backing field)
|
private static void AnalyzeProperties(INamedTypeSymbol typeSymbol, List<PropertyInfo> properties)
|
||||||
bool isReadOnlyGetter = prop.SetMethod == null && !SyntaxHelper.HasBackingField(prop);
|
{
|
||||||
if (isReadOnlyGetter)
|
// Collect properties from the entire inheritance hierarchy
|
||||||
continue;
|
var seenProperties = new HashSet<string>();
|
||||||
|
var currentType = typeSymbol;
|
||||||
|
|
||||||
var columnAttr = AttributeHelper.GetAttribute(prop, "Column");
|
while (currentType != null && currentType.SpecialType != SpecialType.System_Object)
|
||||||
var bsonFieldName = AttributeHelper.GetAttributeStringValue(prop, "BsonProperty") ??
|
{
|
||||||
|
var sourceProps = currentType.GetMembers()
|
||||||
|
.OfType<IPropertySymbol>()
|
||||||
|
.Where(p => p.DeclaredAccessibility == Accessibility.Public && !p.IsStatic);
|
||||||
|
|
||||||
|
foreach (var prop in sourceProps)
|
||||||
|
{
|
||||||
|
// Skip if already seen (overridden property in derived class takes precedence)
|
||||||
|
if (!seenProperties.Add(prop.Name))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (AttributeHelper.ShouldIgnore(prop))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
// Skip computed getter-only properties (no setter, no backing field)
|
||||||
|
bool isReadOnlyGetter = prop.SetMethod == null && !SyntaxHelper.HasBackingField(prop);
|
||||||
|
if (isReadOnlyGetter)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var columnAttr = AttributeHelper.GetAttribute(prop, "Column");
|
||||||
|
string? bsonFieldName = AttributeHelper.GetAttributeStringValue(prop, "BsonProperty") ??
|
||||||
AttributeHelper.GetAttributeStringValue(prop, "JsonPropertyName");
|
AttributeHelper.GetAttributeStringValue(prop, "JsonPropertyName");
|
||||||
|
|
||||||
if (bsonFieldName == null && columnAttr != null)
|
if (bsonFieldName == null && columnAttr != null)
|
||||||
{
|
bsonFieldName = columnAttr.ConstructorArguments.Length > 0
|
||||||
bsonFieldName = columnAttr.ConstructorArguments.Length > 0 ? columnAttr.ConstructorArguments[0].Value?.ToString() : null;
|
? columnAttr.ConstructorArguments[0].Value?.ToString()
|
||||||
}
|
: null;
|
||||||
|
|
||||||
var propInfo = new PropertyInfo
|
var propInfo = new PropertyInfo
|
||||||
{
|
|
||||||
Name = prop.Name,
|
|
||||||
TypeName = SyntaxHelper.GetTypeName(prop.Type),
|
|
||||||
BsonFieldName = bsonFieldName ?? prop.Name.ToLowerInvariant(),
|
|
||||||
ColumnTypeName = columnAttr != null ? AttributeHelper.GetNamedArgumentValue(columnAttr, "TypeName") : null,
|
|
||||||
IsNullable = SyntaxHelper.IsNullableType(prop.Type),
|
|
||||||
IsKey = AttributeHelper.IsKey(prop),
|
|
||||||
IsRequired = AttributeHelper.HasAttribute(prop, "Required"),
|
|
||||||
|
|
||||||
HasPublicSetter = prop.SetMethod?.DeclaredAccessibility == Accessibility.Public,
|
|
||||||
HasInitOnlySetter = prop.SetMethod?.IsInitOnly == true,
|
|
||||||
HasAnySetter = prop.SetMethod != null,
|
|
||||||
IsReadOnlyGetter = isReadOnlyGetter,
|
|
||||||
BackingFieldName = (prop.SetMethod?.DeclaredAccessibility != Accessibility.Public)
|
|
||||||
? $"<{prop.Name}>k__BackingField"
|
|
||||||
: null
|
|
||||||
};
|
|
||||||
|
|
||||||
// MaxLength / MinLength
|
|
||||||
propInfo.MaxLength = AttributeHelper.GetAttributeIntValue(prop, "MaxLength");
|
|
||||||
propInfo.MinLength = AttributeHelper.GetAttributeIntValue(prop, "MinLength");
|
|
||||||
|
|
||||||
var stringLengthAttr = AttributeHelper.GetAttribute(prop, "StringLength");
|
|
||||||
if (stringLengthAttr != null)
|
|
||||||
{
|
|
||||||
if (stringLengthAttr.ConstructorArguments.Length > 0 && stringLengthAttr.ConstructorArguments[0].Value is int max)
|
|
||||||
propInfo.MaxLength = max;
|
|
||||||
|
|
||||||
var minLenStr = AttributeHelper.GetNamedArgumentValue(stringLengthAttr, "MinimumLength");
|
|
||||||
if (int.TryParse(minLenStr, out var min))
|
|
||||||
propInfo.MinLength = min;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Range
|
|
||||||
var rangeAttr = AttributeHelper.GetAttribute(prop, "Range");
|
|
||||||
if (rangeAttr != null && rangeAttr.ConstructorArguments.Length >= 2)
|
|
||||||
{
|
|
||||||
if (rangeAttr.ConstructorArguments[0].Value is double dmin) propInfo.RangeMin = dmin;
|
|
||||||
else if (rangeAttr.ConstructorArguments[0].Value is int imin) propInfo.RangeMin = (double)imin;
|
|
||||||
|
|
||||||
if (rangeAttr.ConstructorArguments[1].Value is double dmax) propInfo.RangeMax = dmax;
|
|
||||||
else if (rangeAttr.ConstructorArguments[1].Value is int imax) propInfo.RangeMax = (double)imax;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (SyntaxHelper.IsCollectionType(prop.Type, out var itemType))
|
|
||||||
{
|
|
||||||
propInfo.IsCollection = true;
|
|
||||||
propInfo.IsArray = prop.Type is IArrayTypeSymbol;
|
|
||||||
|
|
||||||
// Determine concrete collection type name
|
|
||||||
propInfo.CollectionConcreteTypeName = SyntaxHelper.GetTypeName(prop.Type);
|
|
||||||
|
|
||||||
if (itemType != null)
|
|
||||||
{
|
|
||||||
propInfo.CollectionItemType = SyntaxHelper.GetTypeName(itemType);
|
|
||||||
|
|
||||||
// Check if collection item is nested object
|
|
||||||
if (SyntaxHelper.IsNestedObjectType(itemType))
|
|
||||||
{
|
|
||||||
propInfo.IsCollectionItemNested = true;
|
|
||||||
propInfo.NestedTypeName = itemType.Name;
|
|
||||||
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)itemType);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Check for Nested Object
|
|
||||||
else if (SyntaxHelper.IsNestedObjectType(prop.Type))
|
|
||||||
{
|
|
||||||
propInfo.IsNestedObject = true;
|
|
||||||
propInfo.NestedTypeName = prop.Type.Name;
|
|
||||||
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)prop.Type);
|
|
||||||
}
|
|
||||||
|
|
||||||
properties.Add(propInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
currentType = currentType.BaseType;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void AnalyzeNestedTypesRecursive(
|
|
||||||
List<PropertyInfo> properties,
|
|
||||||
Dictionary<string, NestedTypeInfo> targetNestedTypes,
|
|
||||||
SemanticModel semanticModel,
|
|
||||||
HashSet<string> analyzedTypes,
|
|
||||||
int currentDepth,
|
|
||||||
int maxDepth)
|
|
||||||
{
|
|
||||||
if (currentDepth > maxDepth) return;
|
|
||||||
|
|
||||||
// Identify properties that reference nested types (either directly or via collection)
|
|
||||||
var nestedProps = properties
|
|
||||||
.Where(p => (p.IsNestedObject || p.IsCollectionItemNested) && !string.IsNullOrEmpty(p.NestedTypeFullName))
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
foreach (var prop in nestedProps)
|
|
||||||
{
|
|
||||||
var fullTypeName = prop.NestedTypeFullName!;
|
|
||||||
var simpleName = prop.NestedTypeName!;
|
|
||||||
|
|
||||||
// Avoid cycles
|
|
||||||
if (analyzedTypes.Contains(fullTypeName)) continue;
|
|
||||||
|
|
||||||
// If already in target list, skip
|
|
||||||
if (targetNestedTypes.ContainsKey(fullTypeName)) continue;
|
|
||||||
|
|
||||||
// Try to find the symbol
|
|
||||||
INamedTypeSymbol? nestedTypeSymbol = null;
|
|
||||||
|
|
||||||
// Try by full name
|
|
||||||
nestedTypeSymbol = semanticModel.Compilation.GetTypeByMetadataName(fullTypeName);
|
|
||||||
|
|
||||||
// If not found, try to resolve via semantic model (might be in the same compilation)
|
|
||||||
if (nestedTypeSymbol == null)
|
|
||||||
{
|
{
|
||||||
// This is more complex, but usually fullTypeName from ToDisplayString() is traceable.
|
Name = prop.Name,
|
||||||
// For now, let's assume GetTypeByMetadataName works for fully qualified names.
|
TypeName = SyntaxHelper.GetTypeName(prop.Type),
|
||||||
}
|
BsonFieldName = bsonFieldName ?? prop.Name.ToLowerInvariant(),
|
||||||
|
ColumnTypeName = columnAttr != null
|
||||||
|
? AttributeHelper.GetNamedArgumentValue(columnAttr, "TypeName")
|
||||||
|
: null,
|
||||||
|
IsNullable = SyntaxHelper.IsNullableType(prop.Type),
|
||||||
|
IsKey = AttributeHelper.IsKey(prop),
|
||||||
|
IsRequired = AttributeHelper.HasAttribute(prop, "Required"),
|
||||||
|
|
||||||
if (nestedTypeSymbol == null) continue;
|
HasPublicSetter = prop.SetMethod?.DeclaredAccessibility == Accessibility.Public,
|
||||||
|
HasInitOnlySetter = prop.SetMethod?.IsInitOnly == true,
|
||||||
analyzedTypes.Add(fullTypeName);
|
HasAnySetter = prop.SetMethod != null,
|
||||||
|
IsReadOnlyGetter = isReadOnlyGetter,
|
||||||
var nestedInfo = new NestedTypeInfo
|
BackingFieldName = prop.SetMethod?.DeclaredAccessibility != Accessibility.Public
|
||||||
{
|
? $"<{prop.Name}>k__BackingField"
|
||||||
Name = simpleName,
|
: null
|
||||||
Namespace = nestedTypeSymbol.ContainingNamespace.ToDisplayString(),
|
|
||||||
FullTypeName = fullTypeName,
|
|
||||||
Depth = currentDepth
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Analyze properties of this nested type
|
// MaxLength / MinLength
|
||||||
AnalyzeProperties(nestedTypeSymbol, nestedInfo.Properties);
|
propInfo.MaxLength = AttributeHelper.GetAttributeIntValue(prop, "MaxLength");
|
||||||
targetNestedTypes[fullTypeName] = nestedInfo;
|
propInfo.MinLength = AttributeHelper.GetAttributeIntValue(prop, "MinLength");
|
||||||
|
|
||||||
// Recurse
|
var stringLengthAttr = AttributeHelper.GetAttribute(prop, "StringLength");
|
||||||
AnalyzeNestedTypesRecursive(nestedInfo.Properties, nestedInfo.NestedTypes, semanticModel, analyzedTypes, currentDepth + 1, maxDepth);
|
if (stringLengthAttr != null)
|
||||||
}
|
{
|
||||||
}
|
if (stringLengthAttr.ConstructorArguments.Length > 0 &&
|
||||||
}
|
stringLengthAttr.ConstructorArguments[0].Value is int max)
|
||||||
}
|
propInfo.MaxLength = max;
|
||||||
|
|
||||||
|
string? minLenStr = AttributeHelper.GetNamedArgumentValue(stringLengthAttr, "MinimumLength");
|
||||||
|
if (int.TryParse(minLenStr, out int min))
|
||||||
|
propInfo.MinLength = min;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Range
|
||||||
|
var rangeAttr = AttributeHelper.GetAttribute(prop, "Range");
|
||||||
|
if (rangeAttr != null && rangeAttr.ConstructorArguments.Length >= 2)
|
||||||
|
{
|
||||||
|
if (rangeAttr.ConstructorArguments[0].Value is double dmin) propInfo.RangeMin = dmin;
|
||||||
|
else if (rangeAttr.ConstructorArguments[0].Value is int imin) propInfo.RangeMin = (double)imin;
|
||||||
|
|
||||||
|
if (rangeAttr.ConstructorArguments[1].Value is double dmax) propInfo.RangeMax = dmax;
|
||||||
|
else if (rangeAttr.ConstructorArguments[1].Value is int imax) propInfo.RangeMax = (double)imax;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (SyntaxHelper.IsCollectionType(prop.Type, out var itemType))
|
||||||
|
{
|
||||||
|
propInfo.IsCollection = true;
|
||||||
|
propInfo.IsArray = prop.Type is IArrayTypeSymbol;
|
||||||
|
|
||||||
|
// Determine concrete collection type name
|
||||||
|
propInfo.CollectionConcreteTypeName = SyntaxHelper.GetTypeName(prop.Type);
|
||||||
|
|
||||||
|
if (itemType != null)
|
||||||
|
{
|
||||||
|
propInfo.CollectionItemType = SyntaxHelper.GetTypeName(itemType);
|
||||||
|
|
||||||
|
// Check if collection item is nested object
|
||||||
|
if (SyntaxHelper.IsNestedObjectType(itemType))
|
||||||
|
{
|
||||||
|
propInfo.IsCollectionItemNested = true;
|
||||||
|
propInfo.NestedTypeName = itemType.Name;
|
||||||
|
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)itemType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Check for Nested Object
|
||||||
|
else if (SyntaxHelper.IsNestedObjectType(prop.Type))
|
||||||
|
{
|
||||||
|
propInfo.IsNestedObject = true;
|
||||||
|
propInfo.NestedTypeName = prop.Type.Name;
|
||||||
|
propInfo.NestedTypeFullName = SyntaxHelper.GetFullName((INamedTypeSymbol)prop.Type);
|
||||||
|
}
|
||||||
|
|
||||||
|
properties.Add(propInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
currentType = currentType.BaseType;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AnalyzeNestedTypesRecursive(
|
||||||
|
List<PropertyInfo> properties,
|
||||||
|
Dictionary<string, NestedTypeInfo> targetNestedTypes,
|
||||||
|
SemanticModel semanticModel,
|
||||||
|
HashSet<string> analyzedTypes,
|
||||||
|
int currentDepth,
|
||||||
|
int maxDepth)
|
||||||
|
{
|
||||||
|
if (currentDepth > maxDepth) return;
|
||||||
|
|
||||||
|
// Identify properties that reference nested types (either directly or via collection)
|
||||||
|
var nestedProps = properties
|
||||||
|
.Where(p => (p.IsNestedObject || p.IsCollectionItemNested) && !string.IsNullOrEmpty(p.NestedTypeFullName))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
foreach (var prop in nestedProps)
|
||||||
|
{
|
||||||
|
string fullTypeName = prop.NestedTypeFullName!;
|
||||||
|
string simpleName = prop.NestedTypeName!;
|
||||||
|
|
||||||
|
// Avoid cycles
|
||||||
|
if (analyzedTypes.Contains(fullTypeName)) continue;
|
||||||
|
|
||||||
|
// If already in target list, skip
|
||||||
|
if (targetNestedTypes.ContainsKey(fullTypeName)) continue;
|
||||||
|
|
||||||
|
// Try to find the symbol
|
||||||
|
INamedTypeSymbol? nestedTypeSymbol = null;
|
||||||
|
|
||||||
|
// Try by full name
|
||||||
|
nestedTypeSymbol = semanticModel.Compilation.GetTypeByMetadataName(fullTypeName);
|
||||||
|
|
||||||
|
// If not found, try to resolve via semantic model (might be in the same compilation)
|
||||||
|
if (nestedTypeSymbol == null)
|
||||||
|
{
|
||||||
|
// This is more complex, but usually fullTypeName from ToDisplayString() is traceable.
|
||||||
|
// For now, let's assume GetTypeByMetadataName works for fully qualified names.
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nestedTypeSymbol == null) continue;
|
||||||
|
|
||||||
|
analyzedTypes.Add(fullTypeName);
|
||||||
|
|
||||||
|
var nestedInfo = new NestedTypeInfo
|
||||||
|
{
|
||||||
|
Name = simpleName,
|
||||||
|
Namespace = nestedTypeSymbol.ContainingNamespace.ToDisplayString(),
|
||||||
|
FullTypeName = fullTypeName,
|
||||||
|
Depth = currentDepth
|
||||||
|
};
|
||||||
|
|
||||||
|
// Analyze properties of this nested type
|
||||||
|
AnalyzeProperties(nestedTypeSymbol, nestedInfo.Properties);
|
||||||
|
targetNestedTypes[fullTypeName] = nestedInfo;
|
||||||
|
|
||||||
|
// Recurse
|
||||||
|
AnalyzeNestedTypesRecursive(nestedInfo.Properties, nestedInfo.NestedTypes, semanticModel, analyzedTypes,
|
||||||
|
currentDepth + 1, maxDepth);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,403 +1,401 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Microsoft.CodeAnalysis;
|
using Microsoft.CodeAnalysis;
|
||||||
|
using Microsoft.CodeAnalysis.CSharp;
|
||||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||||
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
using ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||||
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
using ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators
|
namespace ZB.MOM.WW.CBDD.SourceGenerators;
|
||||||
{
|
|
||||||
public class DbContextInfo
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the simple class name of the DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public string ClassName { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the fully qualified class name of the DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public string FullClassName => string.IsNullOrEmpty(Namespace) ? ClassName : $"{Namespace}.{ClassName}";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the namespace that contains the DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public string Namespace { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the source file path where the DbContext was found.
|
|
||||||
/// </summary>
|
|
||||||
public string FilePath { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the DbContext is nested.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsNested { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the DbContext is partial.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsPartial { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the DbContext inherits from another DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasBaseDbContext { get; set; } // True if inherits from another DbContext (not DocumentDbContext directly)
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the entities discovered for this DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public List<EntityInfo> Entities { get; set; } = new List<EntityInfo>();
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the collected nested types keyed by full type name.
|
|
||||||
/// </summary>
|
|
||||||
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; set; } = new Dictionary<string, NestedTypeInfo>();
|
|
||||||
}
|
|
||||||
|
|
||||||
[Generator]
|
public class DbContextInfo
|
||||||
public class MapperGenerator : IIncrementalGenerator
|
{
|
||||||
{
|
/// <summary>
|
||||||
/// <summary>
|
/// Gets or sets the simple class name of the DbContext.
|
||||||
/// Initializes the mapper source generator pipeline.
|
/// </summary>
|
||||||
/// </summary>
|
public string ClassName { get; set; } = "";
|
||||||
/// <param name="context">The incremental generator initialization context.</param>
|
|
||||||
public void Initialize(IncrementalGeneratorInitializationContext context)
|
/// <summary>
|
||||||
{
|
/// Gets the fully qualified class name of the DbContext.
|
||||||
// Find all classes that inherit from DocumentDbContext
|
/// </summary>
|
||||||
var dbContextClasses = context.SyntaxProvider
|
public string FullClassName => string.IsNullOrEmpty(Namespace) ? ClassName : $"{Namespace}.{ClassName}";
|
||||||
.CreateSyntaxProvider(
|
|
||||||
predicate: static (node, _) => IsPotentialDbContext(node),
|
/// <summary>
|
||||||
transform: static (ctx, _) => GetDbContextInfo(ctx))
|
/// Gets or sets the namespace that contains the DbContext.
|
||||||
.Where(static context => context is not null)
|
/// </summary>
|
||||||
.Collect()
|
public string Namespace { get; set; } = "";
|
||||||
.SelectMany(static (contexts, _) => contexts.GroupBy(c => c!.FullClassName).Select(g => g.First())!);
|
|
||||||
|
/// <summary>
|
||||||
// Generate code for each DbContext
|
/// Gets or sets the source file path where the DbContext was found.
|
||||||
context.RegisterSourceOutput(dbContextClasses, static (spc, dbContext) =>
|
/// </summary>
|
||||||
|
public string FilePath { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the DbContext is nested.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsNested { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the DbContext is partial.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsPartial { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the DbContext inherits from another DbContext.
|
||||||
|
/// </summary>
|
||||||
|
public bool
|
||||||
|
HasBaseDbContext { get; set; } // True if inherits from another DbContext (not DocumentDbContext directly)
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the entities discovered for this DbContext.
|
||||||
|
/// </summary>
|
||||||
|
public List<EntityInfo> Entities { get; set; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the collected nested types keyed by full type name.
|
||||||
|
/// </summary>
|
||||||
|
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; set; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Generator]
|
||||||
|
public class MapperGenerator : IIncrementalGenerator
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Initializes the mapper source generator pipeline.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="context">The incremental generator initialization context.</param>
|
||||||
|
public void Initialize(IncrementalGeneratorInitializationContext context)
|
||||||
|
{
|
||||||
|
// Find all classes that inherit from DocumentDbContext
|
||||||
|
var dbContextClasses = context.SyntaxProvider
|
||||||
|
.CreateSyntaxProvider(
|
||||||
|
static (node, _) => IsPotentialDbContext(node),
|
||||||
|
static (ctx, _) => GetDbContextInfo(ctx))
|
||||||
|
.Where(static context => context is not null)
|
||||||
|
.Collect()
|
||||||
|
.SelectMany(static (contexts, _) => contexts.GroupBy(c => c!.FullClassName).Select(g => g.First())!);
|
||||||
|
|
||||||
|
// Generate code for each DbContext
|
||||||
|
context.RegisterSourceOutput(dbContextClasses, static (spc, dbContext) =>
|
||||||
|
{
|
||||||
|
if (dbContext == null) return;
|
||||||
|
|
||||||
|
var sb = new StringBuilder();
|
||||||
|
sb.AppendLine($"// Found DbContext: {dbContext.ClassName}");
|
||||||
|
sb.AppendLine(
|
||||||
|
$"// BaseType: {(dbContext.HasBaseDbContext ? "inherits from another DbContext" : "inherits from DocumentDbContext directly")}");
|
||||||
|
|
||||||
|
|
||||||
|
foreach (var entity in dbContext.Entities)
|
||||||
|
// Aggregate nested types recursively
|
||||||
|
CollectNestedTypes(entity.NestedTypes, dbContext.GlobalNestedTypes);
|
||||||
|
|
||||||
|
// Collect namespaces
|
||||||
|
var namespaces = new HashSet<string>
|
||||||
{
|
{
|
||||||
if (dbContext == null) return;
|
"System",
|
||||||
|
"System.Collections.Generic",
|
||||||
|
"ZB.MOM.WW.CBDD.Bson",
|
||||||
|
"ZB.MOM.WW.CBDD.Core.Collections"
|
||||||
|
};
|
||||||
|
|
||||||
|
// Add Entity namespaces
|
||||||
|
foreach (var entity in dbContext.Entities)
|
||||||
|
if (!string.IsNullOrEmpty(entity.Namespace))
|
||||||
|
namespaces.Add(entity.Namespace);
|
||||||
|
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
||||||
|
if (!string.IsNullOrEmpty(nested.Namespace))
|
||||||
|
namespaces.Add(nested.Namespace);
|
||||||
|
|
||||||
|
// Sanitize file path for name uniqueness
|
||||||
|
string safeName = dbContext.ClassName;
|
||||||
|
if (!string.IsNullOrEmpty(dbContext.FilePath))
|
||||||
|
{
|
||||||
|
string fileName = Path.GetFileNameWithoutExtension(dbContext.FilePath);
|
||||||
|
safeName += $"_{fileName}";
|
||||||
|
}
|
||||||
|
|
||||||
|
sb.AppendLine("// <auto-generated/>");
|
||||||
|
sb.AppendLine("#nullable enable");
|
||||||
|
foreach (string ns in namespaces.OrderBy(n => n)) sb.AppendLine($"using {ns};");
|
||||||
|
sb.AppendLine();
|
||||||
|
|
||||||
|
// Use safeName (Context + Filename) to avoid collisions
|
||||||
|
var mapperNamespace = $"{dbContext.Namespace}.{safeName}_Mappers";
|
||||||
|
sb.AppendLine($"namespace {mapperNamespace}");
|
||||||
|
sb.AppendLine("{");
|
||||||
|
|
||||||
|
var generatedMappers = new HashSet<string>();
|
||||||
|
|
||||||
|
// Generate Entity Mappers
|
||||||
|
foreach (var entity in dbContext.Entities)
|
||||||
|
if (generatedMappers.Add(entity.FullTypeName))
|
||||||
|
sb.AppendLine(CodeGenerator.GenerateMapperClass(entity, mapperNamespace));
|
||||||
|
|
||||||
|
// Generate Nested Mappers
|
||||||
|
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
||||||
|
if (generatedMappers.Add(nested.FullTypeName))
|
||||||
|
{
|
||||||
|
var nestedEntity = new EntityInfo
|
||||||
|
{
|
||||||
|
Name = nested.Name,
|
||||||
|
Namespace = nested.Namespace,
|
||||||
|
FullTypeName = nested.FullTypeName // Ensure FullTypeName is copied
|
||||||
|
// Helper to copy properties
|
||||||
|
};
|
||||||
|
nestedEntity.Properties.AddRange(nested.Properties);
|
||||||
|
|
||||||
|
sb.AppendLine(CodeGenerator.GenerateMapperClass(nestedEntity, mapperNamespace));
|
||||||
|
}
|
||||||
|
|
||||||
|
sb.AppendLine("}");
|
||||||
|
sb.AppendLine();
|
||||||
|
|
||||||
|
// Partial DbContext for InitializeCollections (Only for top-level partial classes)
|
||||||
|
if (!dbContext.IsNested && dbContext.IsPartial)
|
||||||
|
{
|
||||||
|
sb.AppendLine($"namespace {dbContext.Namespace}");
|
||||||
|
sb.AppendLine("{");
|
||||||
|
sb.AppendLine($" public partial class {dbContext.ClassName}");
|
||||||
|
sb.AppendLine(" {");
|
||||||
|
sb.AppendLine(" protected override void InitializeCollections()");
|
||||||
|
sb.AppendLine(" {");
|
||||||
|
|
||||||
|
// Call base.InitializeCollections() if this context inherits from another DbContext
|
||||||
|
if (dbContext.HasBaseDbContext) sb.AppendLine(" base.InitializeCollections();");
|
||||||
|
|
||||||
var sb = new StringBuilder();
|
|
||||||
sb.AppendLine($"// Found DbContext: {dbContext.ClassName}");
|
|
||||||
sb.AppendLine($"// BaseType: {(dbContext.HasBaseDbContext ? "inherits from another DbContext" : "inherits from DocumentDbContext directly")}");
|
|
||||||
|
|
||||||
|
|
||||||
foreach (var entity in dbContext.Entities)
|
foreach (var entity in dbContext.Entities)
|
||||||
{
|
if (!string.IsNullOrEmpty(entity.CollectionPropertyName))
|
||||||
// Aggregate nested types recursively
|
{
|
||||||
CollectNestedTypes(entity.NestedTypes, dbContext.GlobalNestedTypes);
|
var mapperName =
|
||||||
}
|
$"global::{mapperNamespace}.{CodeGenerator.GetMapperName(entity.FullTypeName)}";
|
||||||
|
sb.AppendLine(
|
||||||
|
$" this.{entity.CollectionPropertyName} = CreateCollection(new {mapperName}());");
|
||||||
|
}
|
||||||
|
|
||||||
// Collect namespaces
|
sb.AppendLine(" }");
|
||||||
var namespaces = new HashSet<string>
|
|
||||||
{
|
|
||||||
"System",
|
|
||||||
"System.Collections.Generic",
|
|
||||||
"ZB.MOM.WW.CBDD.Bson",
|
|
||||||
"ZB.MOM.WW.CBDD.Core.Collections"
|
|
||||||
};
|
|
||||||
|
|
||||||
// Add Entity namespaces
|
|
||||||
foreach (var entity in dbContext.Entities)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrEmpty(entity.Namespace))
|
|
||||||
namespaces.Add(entity.Namespace);
|
|
||||||
}
|
|
||||||
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrEmpty(nested.Namespace))
|
|
||||||
namespaces.Add(nested.Namespace);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sanitize file path for name uniqueness
|
|
||||||
var safeName = dbContext.ClassName;
|
|
||||||
if (!string.IsNullOrEmpty(dbContext.FilePath))
|
|
||||||
{
|
|
||||||
var fileName = System.IO.Path.GetFileNameWithoutExtension(dbContext.FilePath);
|
|
||||||
safeName += $"_{fileName}";
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.AppendLine("// <auto-generated/>");
|
|
||||||
sb.AppendLine("#nullable enable");
|
|
||||||
foreach (var ns in namespaces.OrderBy(n => n))
|
|
||||||
{
|
|
||||||
sb.AppendLine($"using {ns};");
|
|
||||||
}
|
|
||||||
sb.AppendLine();
|
sb.AppendLine();
|
||||||
|
|
||||||
// Use safeName (Context + Filename) to avoid collisions
|
// Generate Set<TId, T>() override
|
||||||
var mapperNamespace = $"{dbContext.Namespace}.{safeName}_Mappers";
|
var collectionsWithProperties = dbContext.Entities
|
||||||
sb.AppendLine($"namespace {mapperNamespace}");
|
.Where(e => !string.IsNullOrEmpty(e.CollectionPropertyName) &&
|
||||||
sb.AppendLine($"{{");
|
!string.IsNullOrEmpty(e.CollectionIdTypeFullName))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
var generatedMappers = new HashSet<string>();
|
if (collectionsWithProperties.Any())
|
||||||
|
|
||||||
// Generate Entity Mappers
|
|
||||||
foreach (var entity in dbContext.Entities)
|
|
||||||
{
|
{
|
||||||
if (generatedMappers.Add(entity.FullTypeName))
|
sb.AppendLine(
|
||||||
|
" public override global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T> Set<TId, T>()");
|
||||||
|
sb.AppendLine(" {");
|
||||||
|
|
||||||
|
foreach (var entity in collectionsWithProperties)
|
||||||
{
|
{
|
||||||
sb.AppendLine(CodeGenerator.GenerateMapperClass(entity, mapperNamespace));
|
var entityTypeStr = $"global::{entity.FullTypeName}";
|
||||||
|
string? idTypeStr = entity.CollectionIdTypeFullName;
|
||||||
|
sb.AppendLine(
|
||||||
|
$" if (typeof(TId) == typeof({idTypeStr}) && typeof(T) == typeof({entityTypeStr}))");
|
||||||
|
sb.AppendLine(
|
||||||
|
$" return (global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T>)(object)this.{entity.CollectionPropertyName};");
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// Generate Nested Mappers
|
|
||||||
foreach (var nested in dbContext.GlobalNestedTypes.Values)
|
|
||||||
{
|
|
||||||
if (generatedMappers.Add(nested.FullTypeName))
|
|
||||||
{
|
|
||||||
var nestedEntity = new EntityInfo
|
|
||||||
{
|
|
||||||
Name = nested.Name,
|
|
||||||
Namespace = nested.Namespace,
|
|
||||||
FullTypeName = nested.FullTypeName, // Ensure FullTypeName is copied
|
|
||||||
// Helper to copy properties
|
|
||||||
};
|
|
||||||
nestedEntity.Properties.AddRange(nested.Properties);
|
|
||||||
|
|
||||||
sb.AppendLine(CodeGenerator.GenerateMapperClass(nestedEntity, mapperNamespace));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.AppendLine($"}}");
|
|
||||||
sb.AppendLine();
|
|
||||||
|
|
||||||
// Partial DbContext for InitializeCollections (Only for top-level partial classes)
|
|
||||||
if (!dbContext.IsNested && dbContext.IsPartial)
|
|
||||||
{
|
|
||||||
sb.AppendLine($"namespace {dbContext.Namespace}");
|
|
||||||
sb.AppendLine($"{{");
|
|
||||||
sb.AppendLine($" public partial class {dbContext.ClassName}");
|
|
||||||
sb.AppendLine($" {{");
|
|
||||||
sb.AppendLine($" protected override void InitializeCollections()");
|
|
||||||
sb.AppendLine($" {{");
|
|
||||||
|
|
||||||
// Call base.InitializeCollections() if this context inherits from another DbContext
|
|
||||||
if (dbContext.HasBaseDbContext)
|
if (dbContext.HasBaseDbContext)
|
||||||
{
|
sb.AppendLine(" return base.Set<TId, T>();");
|
||||||
sb.AppendLine($" base.InitializeCollections();");
|
else
|
||||||
}
|
sb.AppendLine(
|
||||||
|
" throw new global::System.InvalidOperationException($\"No collection registered for entity type '{typeof(T).Name}' with key type '{typeof(TId).Name}'.\");");
|
||||||
foreach (var entity in dbContext.Entities)
|
|
||||||
{
|
|
||||||
if (!string.IsNullOrEmpty(entity.CollectionPropertyName))
|
|
||||||
{
|
|
||||||
var mapperName = $"global::{mapperNamespace}.{CodeGenerator.GetMapperName(entity.FullTypeName)}";
|
|
||||||
sb.AppendLine($" this.{entity.CollectionPropertyName} = CreateCollection(new {mapperName}());");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.AppendLine($" }}");
|
|
||||||
sb.AppendLine();
|
|
||||||
|
|
||||||
// Generate Set<TId, T>() override
|
sb.AppendLine(" }");
|
||||||
var collectionsWithProperties = dbContext.Entities
|
}
|
||||||
.Where(e => !string.IsNullOrEmpty(e.CollectionPropertyName) && !string.IsNullOrEmpty(e.CollectionIdTypeFullName))
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
if (collectionsWithProperties.Any())
|
sb.AppendLine(" }");
|
||||||
{
|
sb.AppendLine("}");
|
||||||
sb.AppendLine($" public override global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T> Set<TId, T>()");
|
}
|
||||||
sb.AppendLine($" {{");
|
|
||||||
|
|
||||||
foreach (var entity in collectionsWithProperties)
|
spc.AddSource($"{dbContext.Namespace}.{safeName}.Mappers.g.cs", sb.ToString());
|
||||||
{
|
});
|
||||||
var entityTypeStr = $"global::{entity.FullTypeName}";
|
}
|
||||||
var idTypeStr = entity.CollectionIdTypeFullName;
|
|
||||||
sb.AppendLine($" if (typeof(TId) == typeof({idTypeStr}) && typeof(T) == typeof({entityTypeStr}))");
|
|
||||||
sb.AppendLine($" return (global::ZB.MOM.WW.CBDD.Core.Collections.DocumentCollection<TId, T>)(object)this.{entity.CollectionPropertyName};");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dbContext.HasBaseDbContext)
|
private static void CollectNestedTypes(Dictionary<string, NestedTypeInfo> source,
|
||||||
{
|
Dictionary<string, NestedTypeInfo> target)
|
||||||
sb.AppendLine($" return base.Set<TId, T>();");
|
{
|
||||||
}
|
foreach (var kvp in source)
|
||||||
else
|
if (!target.ContainsKey(kvp.Value.FullTypeName))
|
||||||
{
|
|
||||||
sb.AppendLine($" throw new global::System.InvalidOperationException($\"No collection registered for entity type '{{typeof(T).Name}}' with key type '{{typeof(TId).Name}}'.\");");
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.AppendLine($" }}");
|
|
||||||
}
|
|
||||||
|
|
||||||
sb.AppendLine($" }}");
|
|
||||||
sb.AppendLine($"}}");
|
|
||||||
}
|
|
||||||
|
|
||||||
spc.AddSource($"{dbContext.Namespace}.{safeName}.Mappers.g.cs", sb.ToString());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void CollectNestedTypes(Dictionary<string, NestedTypeInfo> source, Dictionary<string, NestedTypeInfo> target)
|
|
||||||
{
|
|
||||||
foreach (var kvp in source)
|
|
||||||
{
|
{
|
||||||
if (!target.ContainsKey(kvp.Value.FullTypeName))
|
target[kvp.Value.FullTypeName] = kvp.Value;
|
||||||
|
CollectNestedTypes(kvp.Value.NestedTypes, target);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void PrintNestedTypes(StringBuilder sb, Dictionary<string, NestedTypeInfo> nestedTypes,
|
||||||
|
string indent)
|
||||||
|
{
|
||||||
|
foreach (var nt in nestedTypes.Values)
|
||||||
|
{
|
||||||
|
sb.AppendLine($"//{indent}- {nt.Name} (Depth: {nt.Depth})");
|
||||||
|
if (nt.Properties.Count > 0)
|
||||||
|
// Print properties for nested type to be sure
|
||||||
|
foreach (var p in nt.Properties)
|
||||||
{
|
{
|
||||||
target[kvp.Value.FullTypeName] = kvp.Value;
|
var flags = new List<string>();
|
||||||
CollectNestedTypes(kvp.Value.NestedTypes, target);
|
if (p.IsCollection) flags.Add($"Collection<{p.CollectionItemType}>");
|
||||||
|
if (p.IsNestedObject) flags.Add($"Nested<{p.NestedTypeName}>");
|
||||||
|
string flagStr = flags.Any() ? $" [{string.Join(", ", flags)}]" : "";
|
||||||
|
sb.AppendLine($"//{indent} - {p.Name}: {p.TypeName}{flagStr}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nt.NestedTypes.Any()) PrintNestedTypes(sb, nt.NestedTypes, indent + " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsPotentialDbContext(SyntaxNode node)
|
||||||
|
{
|
||||||
|
if (node.SyntaxTree.FilePath.EndsWith(".g.cs")) return false;
|
||||||
|
|
||||||
|
return node is ClassDeclarationSyntax classDecl &&
|
||||||
|
classDecl.BaseList != null &&
|
||||||
|
classDecl.Identifier.Text.EndsWith("Context");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DbContextInfo? GetDbContextInfo(GeneratorSyntaxContext context)
|
||||||
|
{
|
||||||
|
var classDecl = (ClassDeclarationSyntax)context.Node;
|
||||||
|
var semanticModel = context.SemanticModel;
|
||||||
|
|
||||||
|
var classSymbol = ModelExtensions.GetDeclaredSymbol(semanticModel, classDecl) as INamedTypeSymbol;
|
||||||
|
if (classSymbol == null) return null;
|
||||||
|
|
||||||
|
if (!SyntaxHelper.InheritsFrom(classSymbol, "DocumentDbContext"))
|
||||||
|
return null;
|
||||||
|
|
||||||
|
// Check if this context inherits from another DbContext (not DocumentDbContext directly)
|
||||||
|
var baseType = classSymbol.BaseType;
|
||||||
|
bool hasBaseDbContext = baseType != null &&
|
||||||
|
baseType.Name != "DocumentDbContext" &&
|
||||||
|
SyntaxHelper.InheritsFrom(baseType, "DocumentDbContext");
|
||||||
|
|
||||||
|
var info = new DbContextInfo
|
||||||
|
{
|
||||||
|
ClassName = classSymbol.Name,
|
||||||
|
Namespace = classSymbol.ContainingNamespace.ToDisplayString(),
|
||||||
|
FilePath = classDecl.SyntaxTree.FilePath,
|
||||||
|
IsNested = classSymbol.ContainingType != null,
|
||||||
|
IsPartial = classDecl.Modifiers.Any(m => m.IsKind(SyntaxKind.PartialKeyword)),
|
||||||
|
HasBaseDbContext = hasBaseDbContext
|
||||||
|
};
|
||||||
|
|
||||||
|
// Analyze OnModelCreating to find entities
|
||||||
|
var onModelCreating = classDecl.Members
|
||||||
|
.OfType<MethodDeclarationSyntax>()
|
||||||
|
.FirstOrDefault(m => m.Identifier.Text == "OnModelCreating");
|
||||||
|
|
||||||
|
if (onModelCreating != null)
|
||||||
|
{
|
||||||
|
var entityCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "Entity");
|
||||||
|
foreach (var call in entityCalls)
|
||||||
|
{
|
||||||
|
string? typeName = SyntaxHelper.GetGenericTypeArgument(call);
|
||||||
|
if (typeName != null)
|
||||||
|
{
|
||||||
|
// Try to find the symbol
|
||||||
|
INamedTypeSymbol? entityType = null;
|
||||||
|
|
||||||
|
// 1. Try by name in current compilation (simple name)
|
||||||
|
var symbols = semanticModel.Compilation.GetSymbolsWithName(typeName);
|
||||||
|
entityType = symbols.OfType<INamedTypeSymbol>().FirstOrDefault();
|
||||||
|
|
||||||
|
// 2. Try by metadata name (if fully qualified)
|
||||||
|
if (entityType == null) entityType = semanticModel.Compilation.GetTypeByMetadataName(typeName);
|
||||||
|
|
||||||
|
if (entityType != null)
|
||||||
|
{
|
||||||
|
// Check for duplicates
|
||||||
|
string fullTypeName = SyntaxHelper.GetFullName(entityType);
|
||||||
|
if (!info.Entities.Any(e => e.FullTypeName == fullTypeName))
|
||||||
|
{
|
||||||
|
var entityInfo = EntityAnalyzer.Analyze(entityType, semanticModel);
|
||||||
|
info.Entities.Add(entityInfo);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void PrintNestedTypes(StringBuilder sb, Dictionary<string, NestedTypeInfo> nestedTypes, string indent)
|
|
||||||
{
|
|
||||||
foreach (var nt in nestedTypes.Values)
|
|
||||||
{
|
|
||||||
sb.AppendLine($"//{indent}- {nt.Name} (Depth: {nt.Depth})");
|
|
||||||
if (nt.Properties.Count > 0)
|
|
||||||
{
|
|
||||||
// Print properties for nested type to be sure
|
|
||||||
foreach (var p in nt.Properties)
|
|
||||||
{
|
|
||||||
var flags = new List<string>();
|
|
||||||
if (p.IsCollection) flags.Add($"Collection<{p.CollectionItemType}>");
|
|
||||||
if (p.IsNestedObject) flags.Add($"Nested<{p.NestedTypeName}>");
|
|
||||||
var flagStr = flags.Any() ? $" [{string.Join(", ", flags)}]" : "";
|
|
||||||
sb.AppendLine($"//{indent} - {p.Name}: {p.TypeName}{flagStr}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nt.NestedTypes.Any())
|
|
||||||
{
|
|
||||||
PrintNestedTypes(sb, nt.NestedTypes, indent + " ");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool IsPotentialDbContext(SyntaxNode node)
|
|
||||||
{
|
|
||||||
if (node.SyntaxTree.FilePath.EndsWith(".g.cs")) return false;
|
|
||||||
|
|
||||||
return node is ClassDeclarationSyntax classDecl &&
|
// Analyze OnModelCreating for HasConversion
|
||||||
classDecl.BaseList != null &&
|
if (onModelCreating != null)
|
||||||
classDecl.Identifier.Text.EndsWith("Context");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static DbContextInfo? GetDbContextInfo(GeneratorSyntaxContext context)
|
|
||||||
{
|
{
|
||||||
var classDecl = (ClassDeclarationSyntax)context.Node;
|
var conversionCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "HasConversion");
|
||||||
var semanticModel = context.SemanticModel;
|
foreach (var call in conversionCalls)
|
||||||
|
|
||||||
var classSymbol = semanticModel.GetDeclaredSymbol(classDecl) as INamedTypeSymbol;
|
|
||||||
if (classSymbol == null) return null;
|
|
||||||
|
|
||||||
if (!SyntaxHelper.InheritsFrom(classSymbol, "DocumentDbContext"))
|
|
||||||
return null;
|
|
||||||
|
|
||||||
// Check if this context inherits from another DbContext (not DocumentDbContext directly)
|
|
||||||
var baseType = classSymbol.BaseType;
|
|
||||||
bool hasBaseDbContext = baseType != null &&
|
|
||||||
baseType.Name != "DocumentDbContext" &&
|
|
||||||
SyntaxHelper.InheritsFrom(baseType, "DocumentDbContext");
|
|
||||||
|
|
||||||
var info = new DbContextInfo
|
|
||||||
{
|
{
|
||||||
ClassName = classSymbol.Name,
|
string? converterName = SyntaxHelper.GetGenericTypeArgument(call);
|
||||||
Namespace = classSymbol.ContainingNamespace.ToDisplayString(),
|
if (converterName == null) continue;
|
||||||
FilePath = classDecl.SyntaxTree.FilePath,
|
|
||||||
IsNested = classSymbol.ContainingType != null,
|
|
||||||
IsPartial = classDecl.Modifiers.Any(m => m.IsKind(Microsoft.CodeAnalysis.CSharp.SyntaxKind.PartialKeyword)),
|
|
||||||
HasBaseDbContext = hasBaseDbContext
|
|
||||||
};
|
|
||||||
|
|
||||||
// Analyze OnModelCreating to find entities
|
|
||||||
var onModelCreating = classDecl.Members
|
|
||||||
.OfType<MethodDeclarationSyntax>()
|
|
||||||
.FirstOrDefault(m => m.Identifier.Text == "OnModelCreating");
|
|
||||||
|
|
||||||
if (onModelCreating != null)
|
|
||||||
{
|
|
||||||
var entityCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "Entity");
|
|
||||||
foreach (var call in entityCalls)
|
|
||||||
{
|
|
||||||
var typeName = SyntaxHelper.GetGenericTypeArgument(call);
|
|
||||||
if (typeName != null)
|
|
||||||
{
|
|
||||||
// Try to find the symbol
|
|
||||||
INamedTypeSymbol? entityType = null;
|
|
||||||
|
|
||||||
// 1. Try by name in current compilation (simple name)
|
|
||||||
var symbols = semanticModel.Compilation.GetSymbolsWithName(typeName);
|
|
||||||
entityType = symbols.OfType<INamedTypeSymbol>().FirstOrDefault();
|
|
||||||
|
|
||||||
// 2. Try by metadata name (if fully qualified)
|
|
||||||
if (entityType == null)
|
|
||||||
{
|
|
||||||
entityType = semanticModel.Compilation.GetTypeByMetadataName(typeName);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (entityType != null)
|
// Trace back: .Property(x => x.Id).HasConversion<T>() or .HasKey(x => x.Id).HasConversion<T>()
|
||||||
|
if (call.Expression is MemberAccessExpressionSyntax
|
||||||
|
{
|
||||||
|
Expression: InvocationExpressionSyntax propertyCall
|
||||||
|
} &&
|
||||||
|
propertyCall.Expression is MemberAccessExpressionSyntax
|
||||||
|
{
|
||||||
|
Name: IdentifierNameSyntax { Identifier: { Text: var propertyMethod } }
|
||||||
|
} &&
|
||||||
|
(propertyMethod == "Property" || propertyMethod == "HasKey"))
|
||||||
|
{
|
||||||
|
string? propertyName =
|
||||||
|
SyntaxHelper.GetPropertyName(propertyCall.ArgumentList.Arguments.FirstOrDefault()?.Expression);
|
||||||
|
if (propertyName == null) continue;
|
||||||
|
|
||||||
|
// Trace further back: Entity<T>().Property(...)
|
||||||
|
if (propertyCall.Expression is MemberAccessExpressionSyntax
|
||||||
{
|
{
|
||||||
// Check for duplicates
|
Expression: InvocationExpressionSyntax entityCall
|
||||||
var fullTypeName = SyntaxHelper.GetFullName(entityType);
|
} &&
|
||||||
if (!info.Entities.Any(e => e.FullTypeName == fullTypeName))
|
entityCall.Expression is MemberAccessExpressionSyntax
|
||||||
|
{
|
||||||
|
Name: GenericNameSyntax { Identifier: { Text: "Entity" } }
|
||||||
|
})
|
||||||
|
{
|
||||||
|
string? entityTypeName = SyntaxHelper.GetGenericTypeArgument(entityCall);
|
||||||
|
if (entityTypeName != null)
|
||||||
|
{
|
||||||
|
var entity = info.Entities.FirstOrDefault(e =>
|
||||||
|
e.Name == entityTypeName || e.FullTypeName.EndsWith("." + entityTypeName));
|
||||||
|
if (entity != null)
|
||||||
{
|
{
|
||||||
var entityInfo = EntityAnalyzer.Analyze(entityType, semanticModel);
|
var prop = entity.Properties.FirstOrDefault(p => p.Name == propertyName);
|
||||||
info.Entities.Add(entityInfo);
|
if (prop != null)
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Analyze OnModelCreating for HasConversion
|
|
||||||
if (onModelCreating != null)
|
|
||||||
{
|
|
||||||
var conversionCalls = SyntaxHelper.FindMethodInvocations(onModelCreating, "HasConversion");
|
|
||||||
foreach (var call in conversionCalls)
|
|
||||||
{
|
|
||||||
var converterName = SyntaxHelper.GetGenericTypeArgument(call);
|
|
||||||
if (converterName == null) continue;
|
|
||||||
|
|
||||||
// Trace back: .Property(x => x.Id).HasConversion<T>() or .HasKey(x => x.Id).HasConversion<T>()
|
|
||||||
if (call.Expression is MemberAccessExpressionSyntax { Expression: InvocationExpressionSyntax propertyCall } &&
|
|
||||||
propertyCall.Expression is MemberAccessExpressionSyntax { Name: IdentifierNameSyntax { Identifier: { Text: var propertyMethod } } } &&
|
|
||||||
(propertyMethod == "Property" || propertyMethod == "HasKey"))
|
|
||||||
{
|
|
||||||
var propertyName = SyntaxHelper.GetPropertyName(propertyCall.ArgumentList.Arguments.FirstOrDefault()?.Expression);
|
|
||||||
if (propertyName == null) continue;
|
|
||||||
|
|
||||||
// Trace further back: Entity<T>().Property(...)
|
|
||||||
if (propertyCall.Expression is MemberAccessExpressionSyntax { Expression: InvocationExpressionSyntax entityCall } &&
|
|
||||||
entityCall.Expression is MemberAccessExpressionSyntax { Name: GenericNameSyntax { Identifier: { Text: "Entity" } } })
|
|
||||||
{
|
|
||||||
var entityTypeName = SyntaxHelper.GetGenericTypeArgument(entityCall);
|
|
||||||
if (entityTypeName != null)
|
|
||||||
{
|
|
||||||
var entity = info.Entities.FirstOrDefault(e => e.Name == entityTypeName || e.FullTypeName.EndsWith("." + entityTypeName));
|
|
||||||
if (entity != null)
|
|
||||||
{
|
{
|
||||||
var prop = entity.Properties.FirstOrDefault(p => p.Name == propertyName);
|
// Resolve TProvider from ValueConverter<TModel, TProvider>
|
||||||
if (prop != null)
|
var converterType =
|
||||||
|
semanticModel.Compilation.GetTypeByMetadataName(converterName) ??
|
||||||
|
semanticModel.Compilation.GetSymbolsWithName(converterName)
|
||||||
|
.OfType<INamedTypeSymbol>().FirstOrDefault();
|
||||||
|
|
||||||
|
prop.ConverterTypeName = converterType != null
|
||||||
|
? SyntaxHelper.GetFullName(converterType)
|
||||||
|
: converterName;
|
||||||
|
|
||||||
|
if (converterType != null && converterType.BaseType != null &&
|
||||||
|
converterType.BaseType.Name == "ValueConverter" &&
|
||||||
|
converterType.BaseType.TypeArguments.Length == 2)
|
||||||
{
|
{
|
||||||
// Resolve TProvider from ValueConverter<TModel, TProvider>
|
prop.ProviderTypeName = converterType.BaseType.TypeArguments[1].Name;
|
||||||
var converterType = semanticModel.Compilation.GetTypeByMetadataName(converterName) ??
|
}
|
||||||
semanticModel.Compilation.GetSymbolsWithName(converterName).OfType<INamedTypeSymbol>().FirstOrDefault();
|
else if (converterType != null)
|
||||||
|
{
|
||||||
prop.ConverterTypeName = converterType != null ? SyntaxHelper.GetFullName(converterType) : converterName;
|
// Fallback: search deeper in base types
|
||||||
|
var converterBaseType = converterType.BaseType;
|
||||||
if (converterType != null && converterType.BaseType != null &&
|
while (converterBaseType != null)
|
||||||
converterType.BaseType.Name == "ValueConverter" &&
|
|
||||||
converterType.BaseType.TypeArguments.Length == 2)
|
|
||||||
{
|
{
|
||||||
prop.ProviderTypeName = converterType.BaseType.TypeArguments[1].Name;
|
if (converterBaseType.Name == "ValueConverter" &&
|
||||||
}
|
converterBaseType.TypeArguments.Length == 2)
|
||||||
else if (converterType != null)
|
|
||||||
{
|
|
||||||
// Fallback: search deeper in base types
|
|
||||||
var converterBaseType = converterType.BaseType;
|
|
||||||
while (converterBaseType != null)
|
|
||||||
{
|
{
|
||||||
if (converterBaseType.Name == "ValueConverter" && converterBaseType.TypeArguments.Length == 2)
|
prop.ProviderTypeName = converterBaseType.TypeArguments[1].Name;
|
||||||
{
|
break;
|
||||||
prop.ProviderTypeName = converterBaseType.TypeArguments[1].Name;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
converterBaseType = converterBaseType.BaseType;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
converterBaseType = converterBaseType.BaseType;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -406,31 +404,28 @@ namespace ZB.MOM.WW.CBDD.SourceGenerators
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Analyze properties to find DocumentCollection<TId, TEntity>
|
// Analyze properties to find DocumentCollection<TId, TEntity>
|
||||||
var properties = classSymbol.GetMembers().OfType<IPropertySymbol>();
|
var properties = classSymbol.GetMembers().OfType<IPropertySymbol>();
|
||||||
foreach (var prop in properties)
|
foreach (var prop in properties)
|
||||||
{
|
if (prop.Type is INamedTypeSymbol namedType &&
|
||||||
if (prop.Type is INamedTypeSymbol namedType &&
|
namedType.OriginalDefinition.Name == "DocumentCollection")
|
||||||
namedType.OriginalDefinition.Name == "DocumentCollection")
|
// Expecting 2 type arguments: TId, TEntity
|
||||||
|
if (namedType.TypeArguments.Length == 2)
|
||||||
{
|
{
|
||||||
// Expecting 2 type arguments: TId, TEntity
|
var entityType = namedType.TypeArguments[1];
|
||||||
if (namedType.TypeArguments.Length == 2)
|
var entityInfo = info.Entities.FirstOrDefault(e => e.FullTypeName == entityType.ToDisplayString());
|
||||||
|
|
||||||
|
// If found, update
|
||||||
|
if (entityInfo != null)
|
||||||
{
|
{
|
||||||
var entityType = namedType.TypeArguments[1];
|
entityInfo.CollectionPropertyName = prop.Name;
|
||||||
var entityInfo = info.Entities.FirstOrDefault(e => e.FullTypeName == entityType.ToDisplayString());
|
entityInfo.CollectionIdTypeFullName = namedType.TypeArguments[0]
|
||||||
|
.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
|
||||||
// If found, update
|
|
||||||
if (entityInfo != null)
|
|
||||||
{
|
|
||||||
entityInfo.CollectionPropertyName = prop.Name;
|
|
||||||
entityInfo.CollectionIdTypeFullName = namedType.TypeArguments[0].ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
return info;
|
||||||
return info;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,121 +1,115 @@
|
|||||||
using System;
|
using System.Linq;
|
||||||
using System.Linq;
|
using Microsoft.CodeAnalysis;
|
||||||
using Microsoft.CodeAnalysis;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers
|
|
||||||
{
|
public static class AttributeHelper
|
||||||
public static class AttributeHelper
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a property should be ignored during mapping.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="property">The property symbol to inspect.</param>
|
||||||
|
/// <returns><see langword="true" /> when the property has an ignore attribute; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool ShouldIgnore(IPropertySymbol property)
|
||||||
{
|
{
|
||||||
/// <summary>
|
return HasAttribute(property, "BsonIgnore") ||
|
||||||
/// Determines whether a property should be ignored during mapping.
|
HasAttribute(property, "JsonIgnore") ||
|
||||||
/// </summary>
|
HasAttribute(property, "NotMapped");
|
||||||
/// <param name="property">The property symbol to inspect.</param>
|
|
||||||
/// <returns><see langword="true"/> when the property has an ignore attribute; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool ShouldIgnore(IPropertySymbol property)
|
|
||||||
{
|
|
||||||
return HasAttribute(property, "BsonIgnore") ||
|
|
||||||
HasAttribute(property, "JsonIgnore") ||
|
|
||||||
HasAttribute(property, "NotMapped");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Determines whether a property is marked as a key.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="property">The property symbol to inspect.</param>
|
|
||||||
/// <returns><see langword="true"/> when the property has a key attribute; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool IsKey(IPropertySymbol property)
|
|
||||||
{
|
|
||||||
return HasAttribute(property, "Key") ||
|
|
||||||
HasAttribute(property, "BsonId");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the first constructor argument value for the specified attribute as a string.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="attributeName">The attribute name to match.</param>
|
|
||||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static string? GetAttributeStringValue(ISymbol symbol, string attributeName)
|
|
||||||
{
|
|
||||||
var attr = GetAttribute(symbol, attributeName);
|
|
||||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
|
||||||
{
|
|
||||||
return attr.ConstructorArguments[0].Value?.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the first constructor argument value for the specified attribute as an integer.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="attributeName">The attribute name to match.</param>
|
|
||||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static int? GetAttributeIntValue(ISymbol symbol, string attributeName)
|
|
||||||
{
|
|
||||||
var attr = GetAttribute(symbol, attributeName);
|
|
||||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
|
||||||
{
|
|
||||||
if (attr.ConstructorArguments[0].Value is int val) return val;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the first constructor argument value for the specified attribute as a double.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="attributeName">The attribute name to match.</param>
|
|
||||||
/// <returns>The attribute value if found; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static double? GetAttributeDoubleValue(ISymbol symbol, string attributeName)
|
|
||||||
{
|
|
||||||
var attr = GetAttribute(symbol, attributeName);
|
|
||||||
if (attr != null && attr.ConstructorArguments.Length > 0)
|
|
||||||
{
|
|
||||||
if (attr.ConstructorArguments[0].Value is double val) return val;
|
|
||||||
if (attr.ConstructorArguments[0].Value is float fval) return (double)fval;
|
|
||||||
if (attr.ConstructorArguments[0].Value is int ival) return (double)ival;
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets a named argument value from an attribute.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="attr">The attribute data.</param>
|
|
||||||
/// <param name="name">The named argument key.</param>
|
|
||||||
/// <returns>The named argument value if present; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static string? GetNamedArgumentValue(AttributeData attr, string name)
|
|
||||||
{
|
|
||||||
return attr.NamedArguments.FirstOrDefault(a => a.Key == name).Value.Value?.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the first attribute that matches the specified name.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="attributeName">The attribute name to match.</param>
|
|
||||||
/// <returns>The matching attribute data if found; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static AttributeData? GetAttribute(ISymbol symbol, string attributeName)
|
|
||||||
{
|
|
||||||
return symbol.GetAttributes().FirstOrDefault(a =>
|
|
||||||
a.AttributeClass != null &&
|
|
||||||
(a.AttributeClass.Name == attributeName ||
|
|
||||||
a.AttributeClass.Name == attributeName + "Attribute"));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Determines whether a symbol has the specified attribute.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="attributeName">The attribute name to match.</param>
|
|
||||||
/// <returns><see langword="true"/> when a matching attribute exists; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool HasAttribute(ISymbol symbol, string attributeName)
|
|
||||||
{
|
|
||||||
return GetAttribute(symbol, attributeName) != null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a property is marked as a key.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="property">The property symbol to inspect.</param>
|
||||||
|
/// <returns><see langword="true" /> when the property has a key attribute; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool IsKey(IPropertySymbol property)
|
||||||
|
{
|
||||||
|
return HasAttribute(property, "Key") ||
|
||||||
|
HasAttribute(property, "BsonId");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the first constructor argument value for the specified attribute as a string.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="attributeName">The attribute name to match.</param>
|
||||||
|
/// <returns>The attribute value if found; otherwise, <see langword="null" />.</returns>
|
||||||
|
public static string? GetAttributeStringValue(ISymbol symbol, string attributeName)
|
||||||
|
{
|
||||||
|
var attr = GetAttribute(symbol, attributeName);
|
||||||
|
if (attr != null && attr.ConstructorArguments.Length > 0) return attr.ConstructorArguments[0].Value?.ToString();
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the first constructor argument value for the specified attribute as an integer.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="attributeName">The attribute name to match.</param>
|
||||||
|
/// <returns>The attribute value if found; otherwise, <see langword="null" />.</returns>
|
||||||
|
public static int? GetAttributeIntValue(ISymbol symbol, string attributeName)
|
||||||
|
{
|
||||||
|
var attr = GetAttribute(symbol, attributeName);
|
||||||
|
if (attr != null && attr.ConstructorArguments.Length > 0)
|
||||||
|
if (attr.ConstructorArguments[0].Value is int val)
|
||||||
|
return val;
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the first constructor argument value for the specified attribute as a double.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="attributeName">The attribute name to match.</param>
|
||||||
|
/// <returns>The attribute value if found; otherwise, <see langword="null" />.</returns>
|
||||||
|
public static double? GetAttributeDoubleValue(ISymbol symbol, string attributeName)
|
||||||
|
{
|
||||||
|
var attr = GetAttribute(symbol, attributeName);
|
||||||
|
if (attr != null && attr.ConstructorArguments.Length > 0)
|
||||||
|
{
|
||||||
|
if (attr.ConstructorArguments[0].Value is double val) return val;
|
||||||
|
if (attr.ConstructorArguments[0].Value is float fval) return (double)fval;
|
||||||
|
if (attr.ConstructorArguments[0].Value is int ival) return (double)ival;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets a named argument value from an attribute.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="attr">The attribute data.</param>
|
||||||
|
/// <param name="name">The named argument key.</param>
|
||||||
|
/// <returns>The named argument value if present; otherwise, <see langword="null" />.</returns>
|
||||||
|
public static string? GetNamedArgumentValue(AttributeData attr, string name)
|
||||||
|
{
|
||||||
|
return attr.NamedArguments.FirstOrDefault(a => a.Key == name).Value.Value?.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the first attribute that matches the specified name.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="attributeName">The attribute name to match.</param>
|
||||||
|
/// <returns>The matching attribute data if found; otherwise, <see langword="null" />.</returns>
|
||||||
|
public static AttributeData? GetAttribute(ISymbol symbol, string attributeName)
|
||||||
|
{
|
||||||
|
return symbol.GetAttributes().FirstOrDefault(a =>
|
||||||
|
a.AttributeClass != null &&
|
||||||
|
(a.AttributeClass.Name == attributeName ||
|
||||||
|
a.AttributeClass.Name == attributeName + "Attribute"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a symbol has the specified attribute.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="attributeName">The attribute name to match.</param>
|
||||||
|
/// <returns><see langword="true" /> when a matching attribute exists; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool HasAttribute(ISymbol symbol, string attributeName)
|
||||||
|
{
|
||||||
|
return GetAttribute(symbol, attributeName) != null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,253 +1,229 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using Microsoft.CodeAnalysis;
|
using Microsoft.CodeAnalysis;
|
||||||
using Microsoft.CodeAnalysis.CSharp;
|
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
||||||
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
|
||||||
|
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers;
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Helpers
|
|
||||||
{
|
public static class SyntaxHelper
|
||||||
public static class SyntaxHelper
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a symbol inherits from a base type with the specified name.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="symbol">The symbol to inspect.</param>
|
||||||
|
/// <param name="baseTypeName">The base type name to match.</param>
|
||||||
|
/// <returns><see langword="true" /> if the symbol inherits from the base type; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool InheritsFrom(INamedTypeSymbol symbol, string baseTypeName)
|
||||||
{
|
{
|
||||||
/// <summary>
|
var current = symbol.BaseType;
|
||||||
/// Determines whether a symbol inherits from a base type with the specified name.
|
while (current != null)
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to inspect.</param>
|
|
||||||
/// <param name="baseTypeName">The base type name to match.</param>
|
|
||||||
/// <returns><see langword="true"/> if the symbol inherits from the base type; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool InheritsFrom(INamedTypeSymbol symbol, string baseTypeName)
|
|
||||||
{
|
{
|
||||||
var current = symbol.BaseType;
|
if (current.Name == baseTypeName)
|
||||||
while (current != null)
|
return true;
|
||||||
{
|
current = current.BaseType;
|
||||||
if (current.Name == baseTypeName)
|
|
||||||
return true;
|
|
||||||
current = current.BaseType;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
return false;
|
||||||
/// Finds method invocations with a matching method name under the provided syntax node.
|
}
|
||||||
/// </summary>
|
|
||||||
/// <param name="node">The root syntax node to search.</param>
|
|
||||||
/// <param name="methodName">The method name to match.</param>
|
|
||||||
/// <returns>A list of matching invocation expressions.</returns>
|
|
||||||
public static List<InvocationExpressionSyntax> FindMethodInvocations(SyntaxNode node, string methodName)
|
|
||||||
{
|
|
||||||
return node.DescendantNodes()
|
|
||||||
.OfType<InvocationExpressionSyntax>()
|
|
||||||
.Where(invocation =>
|
|
||||||
{
|
|
||||||
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess)
|
|
||||||
{
|
|
||||||
return memberAccess.Name.Identifier.Text == methodName;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
})
|
|
||||||
.ToList();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the first generic type argument from an invocation, if present.
|
/// Finds method invocations with a matching method name under the provided syntax node.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="invocation">The invocation to inspect.</param>
|
/// <param name="node">The root syntax node to search.</param>
|
||||||
/// <returns>The generic type argument text, or <see langword="null"/> when not available.</returns>
|
/// <param name="methodName">The method name to match.</param>
|
||||||
public static string? GetGenericTypeArgument(InvocationExpressionSyntax invocation)
|
/// <returns>A list of matching invocation expressions.</returns>
|
||||||
{
|
public static List<InvocationExpressionSyntax> FindMethodInvocations(SyntaxNode node, string methodName)
|
||||||
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess &&
|
{
|
||||||
memberAccess.Name is GenericNameSyntax genericName &&
|
return node.DescendantNodes()
|
||||||
genericName.TypeArgumentList.Arguments.Count > 0)
|
.OfType<InvocationExpressionSyntax>()
|
||||||
{
|
.Where(invocation =>
|
||||||
return genericName.TypeArgumentList.Arguments[0].ToString();
|
{
|
||||||
}
|
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess)
|
||||||
return null;
|
return memberAccess.Name.Identifier.Text == methodName;
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Extracts a property name from an expression.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="expression">The expression to analyze.</param>
|
|
||||||
/// <returns>The property name when resolved; otherwise, <see langword="null"/>.</returns>
|
|
||||||
public static string? GetPropertyName(ExpressionSyntax? expression)
|
|
||||||
{
|
|
||||||
if (expression == null) return null;
|
|
||||||
if (expression is LambdaExpressionSyntax lambda)
|
|
||||||
{
|
|
||||||
return GetPropertyName(lambda.Body as ExpressionSyntax);
|
|
||||||
}
|
|
||||||
if (expression is MemberAccessExpressionSyntax memberAccess)
|
|
||||||
{
|
|
||||||
return memberAccess.Name.Identifier.Text;
|
|
||||||
}
|
|
||||||
if (expression is PrefixUnaryExpressionSyntax prefixUnary && prefixUnary.Operand is MemberAccessExpressionSyntax prefixMember)
|
|
||||||
{
|
|
||||||
return prefixMember.Name.Identifier.Text;
|
|
||||||
}
|
|
||||||
if (expression is PostfixUnaryExpressionSyntax postfixUnary && postfixUnary.Operand is MemberAccessExpressionSyntax postfixMember)
|
|
||||||
{
|
|
||||||
return postfixMember.Name.Identifier.Text;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the fully-qualified type name without the global prefix.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="symbol">The symbol to format.</param>
|
|
||||||
/// <returns>The formatted full type name.</returns>
|
|
||||||
public static string GetFullName(INamedTypeSymbol symbol)
|
|
||||||
{
|
|
||||||
return symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)
|
|
||||||
.Replace("global::", "");
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets a display name for a type symbol.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="type">The type symbol to format.</param>
|
|
||||||
/// <returns>The display name.</returns>
|
|
||||||
public static string GetTypeName(ITypeSymbol type)
|
|
||||||
{
|
|
||||||
if (type is INamedTypeSymbol namedType &&
|
|
||||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
|
||||||
{
|
|
||||||
var underlyingType = namedType.TypeArguments[0];
|
|
||||||
return GetTypeName(underlyingType) + "?";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type is IArrayTypeSymbol arrayType)
|
|
||||||
{
|
|
||||||
return GetTypeName(arrayType.ElementType) + "[]";
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type is INamedTypeSymbol nt && nt.IsTupleType)
|
|
||||||
{
|
|
||||||
return type.ToDisplayString();
|
|
||||||
}
|
|
||||||
|
|
||||||
return type.ToDisplayString();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Determines whether a type is nullable.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="type">The type to evaluate.</param>
|
|
||||||
/// <returns><see langword="true"/> if the type is nullable; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool IsNullableType(ITypeSymbol type)
|
|
||||||
{
|
|
||||||
if (type is INamedTypeSymbol namedType &&
|
|
||||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
|
||||||
{
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return type.NullableAnnotation == NullableAnnotation.Annotated;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Determines whether a type is a collection and returns its item type when available.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="type">The type to evaluate.</param>
|
|
||||||
/// <param name="itemType">When this method returns, contains the collection item type if the type is a collection.</param>
|
|
||||||
/// <returns><see langword="true"/> if the type is a collection; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool IsCollectionType(ITypeSymbol type, out ITypeSymbol? itemType)
|
|
||||||
{
|
|
||||||
itemType = null;
|
|
||||||
|
|
||||||
// Exclude string (it's IEnumerable<char> but not a collection for our purposes)
|
|
||||||
if (type.SpecialType == SpecialType.System_String)
|
|
||||||
return false;
|
return false;
|
||||||
|
})
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
// Handle arrays
|
/// <summary>
|
||||||
if (type is IArrayTypeSymbol arrayType)
|
/// Gets the first generic type argument from an invocation, if present.
|
||||||
{
|
/// </summary>
|
||||||
itemType = arrayType.ElementType;
|
/// <param name="invocation">The invocation to inspect.</param>
|
||||||
return true;
|
/// <returns>The generic type argument text, or <see langword="null" /> when not available.</returns>
|
||||||
}
|
public static string? GetGenericTypeArgument(InvocationExpressionSyntax invocation)
|
||||||
|
{
|
||||||
|
if (invocation.Expression is MemberAccessExpressionSyntax memberAccess &&
|
||||||
|
memberAccess.Name is GenericNameSyntax genericName &&
|
||||||
|
genericName.TypeArgumentList.Arguments.Count > 0)
|
||||||
|
return genericName.TypeArgumentList.Arguments[0].ToString();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
// Check if the type itself is IEnumerable<T>
|
/// <summary>
|
||||||
if (type is INamedTypeSymbol namedType && namedType.IsGenericType)
|
/// Extracts a property name from an expression.
|
||||||
{
|
/// </summary>
|
||||||
var typeDefName = namedType.OriginalDefinition.ToDisplayString();
|
/// <param name="expression">The expression to analyze.</param>
|
||||||
if (typeDefName == "System.Collections.Generic.IEnumerable<T>" && namedType.TypeArguments.Length == 1)
|
/// <returns>The property name when resolved; otherwise, <see langword="null" />.</returns>
|
||||||
{
|
public static string? GetPropertyName(ExpressionSyntax? expression)
|
||||||
itemType = namedType.TypeArguments[0];
|
{
|
||||||
return true;
|
if (expression == null) return null;
|
||||||
}
|
if (expression is LambdaExpressionSyntax lambda) return GetPropertyName(lambda.Body as ExpressionSyntax);
|
||||||
}
|
if (expression is MemberAccessExpressionSyntax memberAccess) return memberAccess.Name.Identifier.Text;
|
||||||
|
if (expression is PrefixUnaryExpressionSyntax prefixUnary &&
|
||||||
|
prefixUnary.Operand is MemberAccessExpressionSyntax prefixMember) return prefixMember.Name.Identifier.Text;
|
||||||
|
if (expression is PostfixUnaryExpressionSyntax postfixUnary &&
|
||||||
|
postfixUnary.Operand is MemberAccessExpressionSyntax postfixMember)
|
||||||
|
return postfixMember.Name.Identifier.Text;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
// Check if the type implements IEnumerable<T> by walking all interfaces
|
/// <summary>
|
||||||
var enumerableInterface = type.AllInterfaces
|
/// Gets the fully-qualified type name without the global prefix.
|
||||||
.FirstOrDefault(i => i.IsGenericType &&
|
/// </summary>
|
||||||
i.OriginalDefinition.ToDisplayString() == "System.Collections.Generic.IEnumerable<T>");
|
/// <param name="symbol">The symbol to format.</param>
|
||||||
|
/// <returns>The formatted full type name.</returns>
|
||||||
|
public static string GetFullName(INamedTypeSymbol symbol)
|
||||||
|
{
|
||||||
|
return symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat)
|
||||||
|
.Replace("global::", "");
|
||||||
|
}
|
||||||
|
|
||||||
if (enumerableInterface != null && enumerableInterface.TypeArguments.Length == 1)
|
/// <summary>
|
||||||
{
|
/// Gets a display name for a type symbol.
|
||||||
itemType = enumerableInterface.TypeArguments[0];
|
/// </summary>
|
||||||
return true;
|
/// <param name="type">The type symbol to format.</param>
|
||||||
}
|
/// <returns>The display name.</returns>
|
||||||
|
public static string GetTypeName(ITypeSymbol type)
|
||||||
|
{
|
||||||
|
if (type is INamedTypeSymbol namedType &&
|
||||||
|
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||||
|
{
|
||||||
|
var underlyingType = namedType.TypeArguments[0];
|
||||||
|
return GetTypeName(underlyingType) + "?";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type is IArrayTypeSymbol arrayType) return GetTypeName(arrayType.ElementType) + "[]";
|
||||||
|
|
||||||
|
if (type is INamedTypeSymbol nt && nt.IsTupleType) return type.ToDisplayString();
|
||||||
|
|
||||||
|
return type.ToDisplayString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a type is nullable.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="type">The type to evaluate.</param>
|
||||||
|
/// <returns><see langword="true" /> if the type is nullable; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool IsNullableType(ITypeSymbol type)
|
||||||
|
{
|
||||||
|
if (type is INamedTypeSymbol namedType &&
|
||||||
|
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||||
|
return true;
|
||||||
|
return type.NullableAnnotation == NullableAnnotation.Annotated;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a type is a collection and returns its item type when available.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="type">The type to evaluate.</param>
|
||||||
|
/// <param name="itemType">When this method returns, contains the collection item type if the type is a collection.</param>
|
||||||
|
/// <returns><see langword="true" /> if the type is a collection; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool IsCollectionType(ITypeSymbol type, out ITypeSymbol? itemType)
|
||||||
|
{
|
||||||
|
itemType = null;
|
||||||
|
|
||||||
|
// Exclude string (it's IEnumerable<char> but not a collection for our purposes)
|
||||||
|
if (type.SpecialType == SpecialType.System_String)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
|
// Handle arrays
|
||||||
|
if (type is IArrayTypeSymbol arrayType)
|
||||||
|
{
|
||||||
|
itemType = arrayType.ElementType;
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
// Check if the type itself is IEnumerable<T>
|
||||||
/// Determines whether a type should be treated as a primitive value.
|
if (type is INamedTypeSymbol namedType && namedType.IsGenericType)
|
||||||
/// </summary>
|
|
||||||
/// <param name="type">The type to evaluate.</param>
|
|
||||||
/// <returns><see langword="true"/> if the type is primitive-like; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool IsPrimitiveType(ITypeSymbol type)
|
|
||||||
{
|
{
|
||||||
if (type is INamedTypeSymbol namedType &&
|
string typeDefName = namedType.OriginalDefinition.ToDisplayString();
|
||||||
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
if (typeDefName == "System.Collections.Generic.IEnumerable<T>" && namedType.TypeArguments.Length == 1)
|
||||||
{
|
{
|
||||||
type = namedType.TypeArguments[0];
|
itemType = namedType.TypeArguments[0];
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (type.SpecialType != SpecialType.None && type.SpecialType != SpecialType.System_Object)
|
|
||||||
return true;
|
|
||||||
|
|
||||||
var typeName = type.Name;
|
|
||||||
if (typeName == "Guid" || typeName == "DateTime" || typeName == "DateTimeOffset" ||
|
|
||||||
typeName == "TimeSpan" || typeName == "DateOnly" || typeName == "TimeOnly" ||
|
|
||||||
typeName == "Decimal" || typeName == "ObjectId")
|
|
||||||
return true;
|
|
||||||
|
|
||||||
if (type.TypeKind == TypeKind.Enum)
|
|
||||||
return true;
|
|
||||||
|
|
||||||
if (type is INamedTypeSymbol nt && nt.IsTupleType)
|
|
||||||
return true;
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
// Check if the type implements IEnumerable<T> by walking all interfaces
|
||||||
/// Determines whether a type should be treated as a nested object.
|
var enumerableInterface = type.AllInterfaces
|
||||||
/// </summary>
|
.FirstOrDefault(i => i.IsGenericType &&
|
||||||
/// <param name="type">The type to evaluate.</param>
|
i.OriginalDefinition.ToDisplayString() == "System.Collections.Generic.IEnumerable<T>");
|
||||||
/// <returns><see langword="true"/> if the type is a nested object; otherwise, <see langword="false"/>.</returns>
|
|
||||||
public static bool IsNestedObjectType(ITypeSymbol type)
|
|
||||||
{
|
|
||||||
if (IsPrimitiveType(type)) return false;
|
|
||||||
if (type.SpecialType == SpecialType.System_String) return false;
|
|
||||||
if (IsCollectionType(type, out _)) return false;
|
|
||||||
if (type.SpecialType == SpecialType.System_Object) return false;
|
|
||||||
if (type is INamedTypeSymbol nt && nt.IsTupleType) return false;
|
|
||||||
|
|
||||||
return type.TypeKind == TypeKind.Class || type.TypeKind == TypeKind.Struct;
|
if (enumerableInterface != null && enumerableInterface.TypeArguments.Length == 1)
|
||||||
|
{
|
||||||
|
itemType = enumerableInterface.TypeArguments[0];
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
return false;
|
||||||
/// Determines whether a property has an associated backing field.
|
}
|
||||||
/// </summary>
|
|
||||||
/// <param name="property">The property to inspect.</param>
|
/// <summary>
|
||||||
/// <returns><see langword="true"/> if a backing field is found; otherwise, <see langword="false"/>.</returns>
|
/// Determines whether a type should be treated as a primitive value.
|
||||||
public static bool HasBackingField(IPropertySymbol property)
|
/// </summary>
|
||||||
{
|
/// <param name="type">The type to evaluate.</param>
|
||||||
// Auto-properties have compiler-generated backing fields
|
/// <returns><see langword="true" /> if the type is primitive-like; otherwise, <see langword="false" />.</returns>
|
||||||
// Check if there's a field with the pattern <PropertyName>k__BackingField
|
public static bool IsPrimitiveType(ITypeSymbol type)
|
||||||
return property.ContainingType.GetMembers()
|
{
|
||||||
.OfType<IFieldSymbol>()
|
if (type is INamedTypeSymbol namedType &&
|
||||||
.Any(f => f.AssociatedSymbol?.Equals(property, SymbolEqualityComparer.Default) == true);
|
namedType.OriginalDefinition.SpecialType == SpecialType.System_Nullable_T)
|
||||||
}
|
type = namedType.TypeArguments[0];
|
||||||
}
|
|
||||||
}
|
if (type.SpecialType != SpecialType.None && type.SpecialType != SpecialType.System_Object)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
string typeName = type.Name;
|
||||||
|
if (typeName == "Guid" || typeName == "DateTime" || typeName == "DateTimeOffset" ||
|
||||||
|
typeName == "TimeSpan" || typeName == "DateOnly" || typeName == "TimeOnly" ||
|
||||||
|
typeName == "Decimal" || typeName == "ObjectId")
|
||||||
|
return true;
|
||||||
|
|
||||||
|
if (type.TypeKind == TypeKind.Enum)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
if (type is INamedTypeSymbol nt && nt.IsTupleType)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a type should be treated as a nested object.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="type">The type to evaluate.</param>
|
||||||
|
/// <returns><see langword="true" /> if the type is a nested object; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool IsNestedObjectType(ITypeSymbol type)
|
||||||
|
{
|
||||||
|
if (IsPrimitiveType(type)) return false;
|
||||||
|
if (type.SpecialType == SpecialType.System_String) return false;
|
||||||
|
if (IsCollectionType(type, out _)) return false;
|
||||||
|
if (type.SpecialType == SpecialType.System_Object) return false;
|
||||||
|
if (type is INamedTypeSymbol nt && nt.IsTupleType) return false;
|
||||||
|
|
||||||
|
return type.TypeKind == TypeKind.Class || type.TypeKind == TypeKind.Struct;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Determines whether a property has an associated backing field.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="property">The property to inspect.</param>
|
||||||
|
/// <returns><see langword="true" /> if a backing field is found; otherwise, <see langword="false" />.</returns>
|
||||||
|
public static bool HasBackingField(IPropertySymbol property)
|
||||||
|
{
|
||||||
|
// Auto-properties have compiler-generated backing fields
|
||||||
|
// Check if there's a field with the pattern <PropertyName>k__BackingField
|
||||||
|
return property.ContainingType.GetMembers()
|
||||||
|
.OfType<IFieldSymbol>()
|
||||||
|
.Any(f => f.AssociatedSymbol?.Equals(property, SymbolEqualityComparer.Default) == true);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,32 +1,31 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models
|
|
||||||
{
|
|
||||||
public class DbContextInfo
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the DbContext class name.
|
|
||||||
/// </summary>
|
|
||||||
public string ClassName { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||||
/// Gets or sets the namespace containing the DbContext.
|
|
||||||
/// </summary>
|
|
||||||
public string Namespace { get; set; } = "";
|
|
||||||
|
|
||||||
/// <summary>
|
public class DbContextInfo
|
||||||
/// Gets or sets the source file path for the DbContext.
|
{
|
||||||
/// </summary>
|
/// <summary>
|
||||||
public string FilePath { get; set; } = "";
|
/// Gets or sets the DbContext class name.
|
||||||
|
/// </summary>
|
||||||
|
public string ClassName { get; set; } = "";
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the entity types discovered for the DbContext.
|
/// Gets or sets the namespace containing the DbContext.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<EntityInfo> Entities { get; } = new List<EntityInfo>();
|
public string Namespace { get; set; } = "";
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets global nested types keyed by type name.
|
/// Gets or sets the source file path for the DbContext.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
public string FilePath { get; set; } = "";
|
||||||
}
|
|
||||||
}
|
/// <summary>
|
||||||
|
/// Gets the entity types discovered for the DbContext.
|
||||||
|
/// </summary>
|
||||||
|
public List<EntityInfo> Entities { get; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets global nested types keyed by type name.
|
||||||
|
/// </summary>
|
||||||
|
public Dictionary<string, NestedTypeInfo> GlobalNestedTypes { get; } = new();
|
||||||
|
}
|
||||||
@@ -1,213 +1,247 @@
|
|||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models
|
namespace ZB.MOM.WW.CBDD.SourceGenerators.Models;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Contains metadata describing an entity discovered by source generation.
|
||||||
|
/// </summary>
|
||||||
|
public class EntityInfo
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Contains metadata describing an entity discovered by source generation.
|
/// Gets or sets the entity name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class EntityInfo
|
public string Name { get; set; } = "";
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the entity name.
|
|
||||||
/// </summary>
|
|
||||||
public string Name { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the entity namespace.
|
|
||||||
/// </summary>
|
|
||||||
public string Namespace { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the fully qualified entity type name.
|
|
||||||
/// </summary>
|
|
||||||
public string FullTypeName { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the collection name for the entity.
|
|
||||||
/// </summary>
|
|
||||||
public string CollectionName { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the collection property name.
|
|
||||||
/// </summary>
|
|
||||||
public string? CollectionPropertyName { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the fully qualified collection identifier type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? CollectionIdTypeFullName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the key property for the entity if one exists.
|
|
||||||
/// </summary>
|
|
||||||
public PropertyInfo? IdProperty => Properties.FirstOrDefault(p => p.IsKey);
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether IDs are automatically generated.
|
|
||||||
/// </summary>
|
|
||||||
public bool AutoId { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the entity uses private setters.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasPrivateSetters { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the entity has a private or missing constructor.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasPrivateOrNoConstructor { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets the entity properties.
|
|
||||||
/// </summary>
|
|
||||||
public List<PropertyInfo> Properties { get; } = new List<PropertyInfo>();
|
|
||||||
/// <summary>
|
|
||||||
/// Gets nested type metadata keyed by type name.
|
|
||||||
/// </summary>
|
|
||||||
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
|
||||||
/// <summary>
|
|
||||||
/// Gets property names that should be ignored by mapping.
|
|
||||||
/// </summary>
|
|
||||||
public HashSet<string> IgnoredProperties { get; } = new HashSet<string>();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Contains metadata describing a mapped property.
|
/// Gets or sets the entity namespace.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class PropertyInfo
|
public string Namespace { get; set; } = "";
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the property name.
|
|
||||||
/// </summary>
|
|
||||||
public string Name { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the property type name.
|
|
||||||
/// </summary>
|
|
||||||
public string TypeName { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the BSON field name.
|
|
||||||
/// </summary>
|
|
||||||
public string BsonFieldName { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the database column type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? ColumnTypeName { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is nullable.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsNullable { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property has a public setter.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasPublicSetter { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property uses an init-only setter.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasInitOnlySetter { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property has any setter.
|
|
||||||
/// </summary>
|
|
||||||
public bool HasAnySetter { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the getter is read-only.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsReadOnlyGetter { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the backing field name if available.
|
|
||||||
/// </summary>
|
|
||||||
public string? BackingFieldName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is the key.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsKey { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is required.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsRequired { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the maximum allowed length.
|
|
||||||
/// </summary>
|
|
||||||
public int? MaxLength { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the minimum allowed length.
|
|
||||||
/// </summary>
|
|
||||||
public int? MinLength { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the minimum allowed range value.
|
|
||||||
/// </summary>
|
|
||||||
public double? RangeMin { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the maximum allowed range value.
|
|
||||||
/// </summary>
|
|
||||||
public double? RangeMax { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is a collection.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsCollection { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is an array.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsArray { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the collection item type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? CollectionItemType { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the concrete collection type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? CollectionConcreteTypeName { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether the property is a nested object.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsNestedObject { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets a value indicating whether collection items are nested objects.
|
|
||||||
/// </summary>
|
|
||||||
public bool IsCollectionItemNested { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the nested type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? NestedTypeName { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the fully qualified nested type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? NestedTypeFullName { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the converter type name.
|
|
||||||
/// </summary>
|
|
||||||
public string? ConverterTypeName { get; set; }
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the provider type name used by the converter.
|
|
||||||
/// </summary>
|
|
||||||
public string? ProviderTypeName { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Contains metadata describing a nested type.
|
/// Gets or sets the fully qualified entity type name.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class NestedTypeInfo
|
public string FullTypeName { get; set; } = "";
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the nested type name.
|
|
||||||
/// </summary>
|
|
||||||
public string Name { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the nested type namespace.
|
|
||||||
/// </summary>
|
|
||||||
public string Namespace { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the fully qualified nested type name.
|
|
||||||
/// </summary>
|
|
||||||
public string FullTypeName { get; set; } = "";
|
|
||||||
/// <summary>
|
|
||||||
/// Gets or sets the depth of the nested type.
|
|
||||||
/// </summary>
|
|
||||||
public int Depth { get; set; }
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the nested type properties.
|
/// Gets or sets the collection name for the entity.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<PropertyInfo> Properties { get; } = new List<PropertyInfo>();
|
public string CollectionName { get; set; } = "";
|
||||||
/// <summary>
|
|
||||||
/// Gets nested type metadata keyed by type name.
|
/// <summary>
|
||||||
/// </summary>
|
/// Gets or sets the collection property name.
|
||||||
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new Dictionary<string, NestedTypeInfo>();
|
/// </summary>
|
||||||
}
|
public string? CollectionPropertyName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the fully qualified collection identifier type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? CollectionIdTypeFullName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the key property for the entity if one exists.
|
||||||
|
/// </summary>
|
||||||
|
public PropertyInfo? IdProperty => Properties.FirstOrDefault(p => p.IsKey);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether IDs are automatically generated.
|
||||||
|
/// </summary>
|
||||||
|
public bool AutoId { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the entity uses private setters.
|
||||||
|
/// </summary>
|
||||||
|
public bool HasPrivateSetters { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the entity has a private or missing constructor.
|
||||||
|
/// </summary>
|
||||||
|
public bool HasPrivateOrNoConstructor { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the entity properties.
|
||||||
|
/// </summary>
|
||||||
|
public List<PropertyInfo> Properties { get; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets nested type metadata keyed by type name.
|
||||||
|
/// </summary>
|
||||||
|
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets property names that should be ignored by mapping.
|
||||||
|
/// </summary>
|
||||||
|
public HashSet<string> IgnoredProperties { get; } = new();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Contains metadata describing a mapped property.
|
||||||
|
/// </summary>
|
||||||
|
public class PropertyInfo
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the property name.
|
||||||
|
/// </summary>
|
||||||
|
public string Name { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the property type name.
|
||||||
|
/// </summary>
|
||||||
|
public string TypeName { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the BSON field name.
|
||||||
|
/// </summary>
|
||||||
|
public string BsonFieldName { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the database column type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? ColumnTypeName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is nullable.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsNullable { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property has a public setter.
|
||||||
|
/// </summary>
|
||||||
|
public bool HasPublicSetter { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property uses an init-only setter.
|
||||||
|
/// </summary>
|
||||||
|
public bool HasInitOnlySetter { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property has any setter.
|
||||||
|
/// </summary>
|
||||||
|
public bool HasAnySetter { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the getter is read-only.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsReadOnlyGetter { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the backing field name if available.
|
||||||
|
/// </summary>
|
||||||
|
public string? BackingFieldName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is the key.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsKey { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is required.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsRequired { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the maximum allowed length.
|
||||||
|
/// </summary>
|
||||||
|
public int? MaxLength { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the minimum allowed length.
|
||||||
|
/// </summary>
|
||||||
|
public int? MinLength { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the minimum allowed range value.
|
||||||
|
/// </summary>
|
||||||
|
public double? RangeMin { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the maximum allowed range value.
|
||||||
|
/// </summary>
|
||||||
|
public double? RangeMax { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is a collection.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsCollection { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is an array.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsArray { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the collection item type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? CollectionItemType { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the concrete collection type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? CollectionConcreteTypeName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether the property is a nested object.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsNestedObject { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets a value indicating whether collection items are nested objects.
|
||||||
|
/// </summary>
|
||||||
|
public bool IsCollectionItemNested { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the nested type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? NestedTypeName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the fully qualified nested type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? NestedTypeFullName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the converter type name.
|
||||||
|
/// </summary>
|
||||||
|
public string? ConverterTypeName { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the provider type name used by the converter.
|
||||||
|
/// </summary>
|
||||||
|
public string? ProviderTypeName { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Contains metadata describing a nested type.
|
||||||
|
/// </summary>
|
||||||
|
public class NestedTypeInfo
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the nested type name.
|
||||||
|
/// </summary>
|
||||||
|
public string Name { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the nested type namespace.
|
||||||
|
/// </summary>
|
||||||
|
public string Namespace { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the fully qualified nested type name.
|
||||||
|
/// </summary>
|
||||||
|
public string FullTypeName { get; set; } = "";
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets or sets the depth of the nested type.
|
||||||
|
/// </summary>
|
||||||
|
public int Depth { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the nested type properties.
|
||||||
|
/// </summary>
|
||||||
|
public List<PropertyInfo> Properties { get; } = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets nested type metadata keyed by type name.
|
||||||
|
/// </summary>
|
||||||
|
public Dictionary<string, NestedTypeInfo> NestedTypes { get; } = new();
|
||||||
|
}
|
||||||
@@ -1,38 +1,38 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
<AssemblyName>ZB.MOM.WW.CBDD.SourceGenerators</AssemblyName>
|
||||||
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
<RootNamespace>ZB.MOM.WW.CBDD.SourceGenerators</RootNamespace>
|
||||||
<LangVersion>latest</LangVersion>
|
<LangVersion>latest</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
|
<EnforceExtendedAnalyzerRules>true</EnforceExtendedAnalyzerRules>
|
||||||
<IsRoslynComponent>true</IsRoslynComponent>
|
<IsRoslynComponent>true</IsRoslynComponent>
|
||||||
<PackageId>ZB.MOM.WW.CBDD.SourceGenerators</PackageId>
|
<PackageId>ZB.MOM.WW.CBDD.SourceGenerators</PackageId>
|
||||||
<Version>1.3.1</Version>
|
<Version>1.3.1</Version>
|
||||||
<Authors>CBDD Team</Authors>
|
<Authors>CBDD Team</Authors>
|
||||||
<Description>Source Generators for CBDD High-Performance BSON Database Engine</Description>
|
<Description>Source Generators for CBDD High-Performance BSON Database Engine</Description>
|
||||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation;source-generator</PackageTags>
|
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation;source-generator</PackageTags>
|
||||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||||
<IncludeBuildOutput>false</IncludeBuildOutput>
|
<IncludeBuildOutput>false</IncludeBuildOutput>
|
||||||
<DevelopmentDependency>true</DevelopmentDependency>
|
<DevelopmentDependency>true</DevelopmentDependency>
|
||||||
<NoPackageAnalysis>true</NoPackageAnalysis>
|
<NoPackageAnalysis>true</NoPackageAnalysis>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" PrivateAssets="all" />
|
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" PrivateAssets="all"/>
|
||||||
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" PrivateAssets="all" />
|
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.3.4" PrivateAssets="all"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false" />
|
<None Include="$(OutputPath)\$(AssemblyName).dll" Pack="true" PackagePath="analyzers/dotnet/cs" Visible="false"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
<None Include="..\..\README.md" Pack="true" PackagePath="\"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,34 +1,34 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<AssemblyName>ZB.MOM.WW.CBDD</AssemblyName>
|
<AssemblyName>ZB.MOM.WW.CBDD</AssemblyName>
|
||||||
<RootNamespace>ZB.MOM.WW.CBDD</RootNamespace>
|
<RootNamespace>ZB.MOM.WW.CBDD</RootNamespace>
|
||||||
<LangVersion>latest</LangVersion>
|
<LangVersion>latest</LangVersion>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||||
|
|
||||||
<PackageId>ZB.MOM.WW.CBDD</PackageId>
|
|
||||||
<Version>1.3.1</Version>
|
|
||||||
<Authors>CBDD Team</Authors>
|
|
||||||
<Description>High-Performance BSON Database Engine for .NET 10</Description>
|
|
||||||
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
|
||||||
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
|
||||||
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
|
||||||
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
<PackageId>ZB.MOM.WW.CBDD</PackageId>
|
||||||
<ProjectReference Include="..\CBDD.Core\ZB.MOM.WW.CBDD.Core.csproj" />
|
<Version>1.3.1</Version>
|
||||||
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj" />
|
<Authors>CBDD Team</Authors>
|
||||||
<ProjectReference Include="..\CBDD.SourceGenerators\ZB.MOM.WW.CBDD.SourceGenerators.csproj" PrivateAssets="none" />
|
<Description>High-Performance BSON Database Engine for .NET 10</Description>
|
||||||
</ItemGroup>
|
<PackageLicenseExpression>MIT</PackageLicenseExpression>
|
||||||
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
|
<RepositoryUrl>https://github.com/EntglDb/CBDD</RepositoryUrl>
|
||||||
|
<PackageTags>database;embedded;bson;nosql;net10;zero-allocation</PackageTags>
|
||||||
|
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
|
<ProjectReference Include="..\CBDD.Core\ZB.MOM.WW.CBDD.Core.csproj"/>
|
||||||
</ItemGroup>
|
<ProjectReference Include="..\CBDD.Bson\ZB.MOM.WW.CBDD.Bson.csproj"/>
|
||||||
|
<ProjectReference Include="..\CBDD.SourceGenerators\ZB.MOM.WW.CBDD.SourceGenerators.csproj" PrivateAssets="none"/>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<None Include="..\..\README.md" Pack="true" PackagePath="\"/>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
|
using System.Text;
|
||||||
using BenchmarkDotNet.Attributes;
|
using BenchmarkDotNet.Attributes;
|
||||||
using BenchmarkDotNet.Configs;
|
using BenchmarkDotNet.Configs;
|
||||||
using BenchmarkDotNet.Jobs;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Core.Storage;
|
using ZB.MOM.WW.CBDD.Core.Storage;
|
||||||
@@ -15,21 +15,22 @@ namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
|||||||
[JsonExporterAttribute.Full]
|
[JsonExporterAttribute.Full]
|
||||||
public class CompactionBenchmarks
|
public class CompactionBenchmarks
|
||||||
{
|
{
|
||||||
|
private readonly List<ObjectId> _insertedIds = [];
|
||||||
|
private DocumentCollection<Person> _collection = null!;
|
||||||
|
|
||||||
|
private string _dbPath = string.Empty;
|
||||||
|
private StorageEngine _storage = null!;
|
||||||
|
private BenchmarkTransactionHolder _transactionHolder = null!;
|
||||||
|
private string _walPath = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the number of documents used per benchmark iteration.
|
/// Gets or sets the number of documents used per benchmark iteration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Params(2_000)]
|
[Params(2_000)]
|
||||||
public int DocumentCount { get; set; }
|
public int DocumentCount { get; set; }
|
||||||
|
|
||||||
private string _dbPath = string.Empty;
|
|
||||||
private string _walPath = string.Empty;
|
|
||||||
private StorageEngine _storage = null!;
|
|
||||||
private BenchmarkTransactionHolder _transactionHolder = null!;
|
|
||||||
private DocumentCollection<Person> _collection = null!;
|
|
||||||
private List<ObjectId> _insertedIds = [];
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prepares benchmark state and seed data for each iteration.
|
/// Prepares benchmark state and seed data for each iteration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[IterationSetup]
|
[IterationSetup]
|
||||||
public void Setup()
|
public void Setup()
|
||||||
@@ -53,17 +54,14 @@ public class CompactionBenchmarks
|
|||||||
_transactionHolder.CommitAndReset();
|
_transactionHolder.CommitAndReset();
|
||||||
_storage.Checkpoint();
|
_storage.Checkpoint();
|
||||||
|
|
||||||
for (var i = _insertedIds.Count - 1; i >= _insertedIds.Count / 3; i--)
|
for (int i = _insertedIds.Count - 1; i >= _insertedIds.Count / 3; i--) _collection.Delete(_insertedIds[i]);
|
||||||
{
|
|
||||||
_collection.Delete(_insertedIds[i]);
|
|
||||||
}
|
|
||||||
|
|
||||||
_transactionHolder.CommitAndReset();
|
_transactionHolder.CommitAndReset();
|
||||||
_storage.Checkpoint();
|
_storage.Checkpoint();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Cleans up benchmark resources and temporary files after each iteration.
|
/// Cleans up benchmark resources and temporary files after each iteration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[IterationCleanup]
|
[IterationCleanup]
|
||||||
public void Cleanup()
|
public void Cleanup()
|
||||||
@@ -76,7 +74,7 @@ public class CompactionBenchmarks
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Benchmarks reclaimed file bytes reported by offline compaction.
|
/// Benchmarks reclaimed file bytes reported by offline compaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The reclaimed file byte count.</returns>
|
/// <returns>The reclaimed file byte count.</returns>
|
||||||
[Benchmark(Baseline = true)]
|
[Benchmark(Baseline = true)]
|
||||||
@@ -95,7 +93,7 @@ public class CompactionBenchmarks
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Benchmarks tail bytes truncated by offline compaction.
|
/// Benchmarks tail bytes truncated by offline compaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>The truncated tail byte count.</returns>
|
/// <returns>The truncated tail byte count.</returns>
|
||||||
[Benchmark]
|
[Benchmark]
|
||||||
@@ -135,7 +133,7 @@ public class CompactionBenchmarks
|
|||||||
|
|
||||||
private static string BuildPayload(int seed)
|
private static string BuildPayload(int seed)
|
||||||
{
|
{
|
||||||
var builder = new System.Text.StringBuilder(2500);
|
var builder = new StringBuilder(2500);
|
||||||
for (var i = 0; i < 80; i++)
|
for (var i = 0; i < 80; i++)
|
||||||
{
|
{
|
||||||
builder.Append("compact-");
|
builder.Append("compact-");
|
||||||
@@ -147,4 +145,4 @@ public class CompactionBenchmarks
|
|||||||
|
|
||||||
return builder.ToString();
|
return builder.ToString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
|
using System.IO.Compression;
|
||||||
|
using System.Text;
|
||||||
using BenchmarkDotNet.Attributes;
|
using BenchmarkDotNet.Attributes;
|
||||||
using BenchmarkDotNet.Configs;
|
using BenchmarkDotNet.Configs;
|
||||||
using BenchmarkDotNet.Jobs;
|
|
||||||
using System.IO.Compression;
|
|
||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using ZB.MOM.WW.CBDD.Core.Compression;
|
using ZB.MOM.WW.CBDD.Core.Compression;
|
||||||
@@ -19,36 +19,36 @@ public class CompressionBenchmarks
|
|||||||
{
|
{
|
||||||
private const int SeedCount = 300;
|
private const int SeedCount = 300;
|
||||||
private const int WorkloadCount = 100;
|
private const int WorkloadCount = 100;
|
||||||
|
private DocumentCollection<Person> _collection = null!;
|
||||||
|
|
||||||
|
private string _dbPath = string.Empty;
|
||||||
|
|
||||||
|
private Person[] _insertBatch = Array.Empty<Person>();
|
||||||
|
private ObjectId[] _seedIds = Array.Empty<ObjectId>();
|
||||||
|
private StorageEngine _storage = null!;
|
||||||
|
private BenchmarkTransactionHolder _transactionHolder = null!;
|
||||||
|
private string _walPath = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets whether compression is enabled for the benchmark run.
|
/// Gets or sets whether compression is enabled for the benchmark run.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Params(false, true)]
|
[Params(false, true)]
|
||||||
public bool EnableCompression { get; set; }
|
public bool EnableCompression { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the compression codec for the benchmark run.
|
/// Gets or sets the compression codec for the benchmark run.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Params(CompressionCodec.Brotli, CompressionCodec.Deflate)]
|
[Params(CompressionCodec.Brotli, CompressionCodec.Deflate)]
|
||||||
public CompressionCodec Codec { get; set; }
|
public CompressionCodec Codec { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the compression level for the benchmark run.
|
/// Gets or sets the compression level for the benchmark run.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Params(CompressionLevel.Fastest, CompressionLevel.Optimal)]
|
[Params(CompressionLevel.Fastest, CompressionLevel.Optimal)]
|
||||||
public CompressionLevel Level { get; set; }
|
public CompressionLevel Level { get; set; }
|
||||||
|
|
||||||
private string _dbPath = string.Empty;
|
|
||||||
private string _walPath = string.Empty;
|
|
||||||
private StorageEngine _storage = null!;
|
|
||||||
private BenchmarkTransactionHolder _transactionHolder = null!;
|
|
||||||
private DocumentCollection<Person> _collection = null!;
|
|
||||||
|
|
||||||
private Person[] _insertBatch = Array.Empty<Person>();
|
|
||||||
private ObjectId[] _seedIds = Array.Empty<ObjectId>();
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Prepares benchmark storage and seed data for each iteration.
|
/// Prepares benchmark storage and seed data for each iteration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[IterationSetup]
|
[IterationSetup]
|
||||||
public void Setup()
|
public void Setup()
|
||||||
@@ -73,19 +73,19 @@ public class CompressionBenchmarks
|
|||||||
_seedIds = new ObjectId[SeedCount];
|
_seedIds = new ObjectId[SeedCount];
|
||||||
for (var i = 0; i < SeedCount; i++)
|
for (var i = 0; i < SeedCount; i++)
|
||||||
{
|
{
|
||||||
var doc = CreatePerson(i, includeLargeBio: true);
|
var doc = CreatePerson(i, true);
|
||||||
_seedIds[i] = _collection.Insert(doc);
|
_seedIds[i] = _collection.Insert(doc);
|
||||||
}
|
}
|
||||||
|
|
||||||
_transactionHolder.CommitAndReset();
|
_transactionHolder.CommitAndReset();
|
||||||
|
|
||||||
_insertBatch = Enumerable.Range(SeedCount, WorkloadCount)
|
_insertBatch = Enumerable.Range(SeedCount, WorkloadCount)
|
||||||
.Select(i => CreatePerson(i, includeLargeBio: true))
|
.Select(i => CreatePerson(i, true))
|
||||||
.ToArray();
|
.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Cleans up benchmark resources for each iteration.
|
/// Cleans up benchmark resources for each iteration.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[IterationCleanup]
|
[IterationCleanup]
|
||||||
public void Cleanup()
|
public void Cleanup()
|
||||||
@@ -98,7 +98,7 @@ public class CompressionBenchmarks
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Benchmarks insert workload performance.
|
/// Benchmarks insert workload performance.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Benchmark(Baseline = true)]
|
[Benchmark(Baseline = true)]
|
||||||
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
||||||
@@ -109,7 +109,7 @@ public class CompressionBenchmarks
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Benchmarks update workload performance.
|
/// Benchmarks update workload performance.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Benchmark]
|
[Benchmark]
|
||||||
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
||||||
@@ -131,7 +131,7 @@ public class CompressionBenchmarks
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Benchmarks read workload performance.
|
/// Benchmarks read workload performance.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[Benchmark]
|
[Benchmark]
|
||||||
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
[BenchmarkCategory("Compression_InsertUpdateRead")]
|
||||||
@@ -141,10 +141,7 @@ public class CompressionBenchmarks
|
|||||||
for (var i = 0; i < WorkloadCount; i++)
|
for (var i = 0; i < WorkloadCount; i++)
|
||||||
{
|
{
|
||||||
var person = _collection.FindById(_seedIds[i]);
|
var person = _collection.FindById(_seedIds[i]);
|
||||||
if (person != null)
|
if (person != null) checksum += person.Age;
|
||||||
{
|
|
||||||
checksum += person.Age;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
_transactionHolder.CommitAndReset();
|
_transactionHolder.CommitAndReset();
|
||||||
@@ -158,7 +155,7 @@ public class CompressionBenchmarks
|
|||||||
Id = ObjectId.NewObjectId(),
|
Id = ObjectId.NewObjectId(),
|
||||||
FirstName = $"First_{i}",
|
FirstName = $"First_{i}",
|
||||||
LastName = $"Last_{i}",
|
LastName = $"Last_{i}",
|
||||||
Age = 20 + (i % 50),
|
Age = 20 + i % 50,
|
||||||
Bio = includeLargeBio ? BuildBio(i) : $"bio-{i}",
|
Bio = includeLargeBio ? BuildBio(i) : $"bio-{i}",
|
||||||
CreatedAt = DateTime.UnixEpoch.AddMinutes(i),
|
CreatedAt = DateTime.UnixEpoch.AddMinutes(i),
|
||||||
Balance = 100 + i,
|
Balance = 100 + i,
|
||||||
@@ -183,7 +180,7 @@ public class CompressionBenchmarks
|
|||||||
|
|
||||||
private static string BuildBio(int seed)
|
private static string BuildBio(int seed)
|
||||||
{
|
{
|
||||||
var builder = new System.Text.StringBuilder(4500);
|
var builder = new StringBuilder(4500);
|
||||||
for (var i = 0; i < 150; i++)
|
for (var i = 0; i < 150; i++)
|
||||||
{
|
{
|
||||||
builder.Append("bio-");
|
builder.Append("bio-");
|
||||||
@@ -195,4 +192,4 @@ public class CompressionBenchmarks
|
|||||||
|
|
||||||
return builder.ToString();
|
return builder.ToString();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,21 +1,21 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
||||||
|
|
||||||
public class Address
|
public class Address
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Street.
|
/// Gets or sets the Street.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string Street { get; set; } = string.Empty;
|
public string Street { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the City.
|
/// Gets or sets the City.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string City { get; set; } = string.Empty;
|
public string City { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the ZipCode.
|
/// Gets or sets the ZipCode.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string ZipCode { get; set; } = string.Empty;
|
public string ZipCode { get; set; } = string.Empty;
|
||||||
}
|
}
|
||||||
@@ -23,19 +23,22 @@ public class Address
|
|||||||
public class WorkHistory
|
public class WorkHistory
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the CompanyName.
|
/// Gets or sets the CompanyName.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string CompanyName { get; set; } = string.Empty;
|
public string CompanyName { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Title.
|
/// Gets or sets the Title.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string Title { get; set; } = string.Empty;
|
public string Title { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the DurationYears.
|
/// Gets or sets the DurationYears.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int DurationYears { get; set; }
|
public int DurationYears { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Tags.
|
/// Gets or sets the Tags.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<string> Tags { get; set; } = new();
|
public List<string> Tags { get; set; } = new();
|
||||||
}
|
}
|
||||||
@@ -43,41 +46,48 @@ public class WorkHistory
|
|||||||
public class Person
|
public class Person
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Id.
|
/// Gets or sets the Id.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public ObjectId Id { get; set; }
|
public ObjectId Id { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the FirstName.
|
/// Gets or sets the FirstName.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string FirstName { get; set; } = string.Empty;
|
public string FirstName { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the LastName.
|
/// Gets or sets the LastName.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string LastName { get; set; } = string.Empty;
|
public string LastName { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Age.
|
/// Gets or sets the Age.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public int Age { get; set; }
|
public int Age { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Bio.
|
/// Gets or sets the Bio.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public string? Bio { get; set; } = string.Empty;
|
public string? Bio { get; set; } = string.Empty;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the CreatedAt.
|
/// Gets or sets the CreatedAt.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public DateTime CreatedAt { get; set; }
|
public DateTime CreatedAt { get; set; }
|
||||||
|
|
||||||
// Complex fields
|
// Complex fields
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the Balance.
|
/// Gets or sets the Balance.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public decimal Balance { get; set; }
|
public decimal Balance { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the HomeAddress.
|
/// Gets or sets the HomeAddress.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public Address HomeAddress { get; set; } = new();
|
public Address HomeAddress { get; set; } = new();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the EmploymentHistory.
|
/// Gets or sets the EmploymentHistory.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public List<WorkHistory> EmploymentHistory { get; set; } = new();
|
public List<WorkHistory> EmploymentHistory { get; set; } = new();
|
||||||
}
|
}
|
||||||
@@ -1,26 +1,30 @@
|
|||||||
using ZB.MOM.WW.CBDD.Bson;
|
using ZB.MOM.WW.CBDD.Bson;
|
||||||
using ZB.MOM.WW.CBDD.Core.Collections;
|
using ZB.MOM.WW.CBDD.Core.Collections;
|
||||||
using System.Buffers;
|
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
|
|
||||||
namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
namespace ZB.MOM.WW.CBDD.Tests.Benchmark;
|
||||||
|
|
||||||
public class PersonMapper : ObjectIdMapperBase<Person>
|
public class PersonMapper : ObjectIdMapperBase<Person>
|
||||||
{
|
{
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override string CollectionName => "people";
|
public override string CollectionName => "people";
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override ObjectId GetId(Person entity) => entity.Id;
|
public override ObjectId GetId(Person entity)
|
||||||
|
{
|
||||||
/// <inheritdoc />
|
return entity.Id;
|
||||||
public override void SetId(Person entity, ObjectId id) => entity.Id = id;
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override int Serialize(Person entity, BsonSpanWriter writer)
|
public override void SetId(Person entity, ObjectId id)
|
||||||
{
|
{
|
||||||
var sizePos = writer.BeginDocument();
|
entity.Id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public override int Serialize(Person entity, BsonSpanWriter writer)
|
||||||
|
{
|
||||||
|
int sizePos = writer.BeginDocument();
|
||||||
|
|
||||||
writer.WriteObjectId("_id", entity.Id);
|
writer.WriteObjectId("_id", entity.Id);
|
||||||
writer.WriteString("firstname", entity.FirstName);
|
writer.WriteString("firstname", entity.FirstName);
|
||||||
writer.WriteString("lastname", entity.LastName);
|
writer.WriteString("lastname", entity.LastName);
|
||||||
@@ -30,111 +34,119 @@ public class PersonMapper : ObjectIdMapperBase<Person>
|
|||||||
else
|
else
|
||||||
writer.WriteNull("bio");
|
writer.WriteNull("bio");
|
||||||
|
|
||||||
writer.WriteInt64("createdat", entity.CreatedAt.Ticks);
|
writer.WriteInt64("createdat", entity.CreatedAt.Ticks);
|
||||||
|
|
||||||
// Complex fields
|
// Complex fields
|
||||||
writer.WriteDouble("balance", (double)entity.Balance);
|
writer.WriteDouble("balance", (double)entity.Balance);
|
||||||
|
|
||||||
// Nested Object: Address
|
// Nested Object: Address
|
||||||
var addrPos = writer.BeginDocument("homeaddress");
|
int addrPos = writer.BeginDocument("homeaddress");
|
||||||
writer.WriteString("street", entity.HomeAddress.Street);
|
writer.WriteString("street", entity.HomeAddress.Street);
|
||||||
writer.WriteString("city", entity.HomeAddress.City);
|
writer.WriteString("city", entity.HomeAddress.City);
|
||||||
writer.WriteString("zipcode", entity.HomeAddress.ZipCode);
|
writer.WriteString("zipcode", entity.HomeAddress.ZipCode);
|
||||||
writer.EndDocument(addrPos);
|
writer.EndDocument(addrPos);
|
||||||
|
|
||||||
// Collection: EmploymentHistory
|
// Collection: EmploymentHistory
|
||||||
var histPos = writer.BeginArray("employmenthistory");
|
int histPos = writer.BeginArray("employmenthistory");
|
||||||
for (int i = 0; i < entity.EmploymentHistory.Count; i++)
|
for (var i = 0; i < entity.EmploymentHistory.Count; i++)
|
||||||
{
|
{
|
||||||
var item = entity.EmploymentHistory[i];
|
var item = entity.EmploymentHistory[i];
|
||||||
// Array elements are keys "0", "1", "2"...
|
// Array elements are keys "0", "1", "2"...
|
||||||
var itemPos = writer.BeginDocument(i.ToString());
|
int itemPos = writer.BeginDocument(i.ToString());
|
||||||
|
|
||||||
writer.WriteString("companyname", item.CompanyName);
|
writer.WriteString("companyname", item.CompanyName);
|
||||||
writer.WriteString("title", item.Title);
|
writer.WriteString("title", item.Title);
|
||||||
writer.WriteInt32("durationyears", item.DurationYears);
|
writer.WriteInt32("durationyears", item.DurationYears);
|
||||||
|
|
||||||
// Nested Collection: Tags
|
// Nested Collection: Tags
|
||||||
var tagsPos = writer.BeginArray("tags");
|
int tagsPos = writer.BeginArray("tags");
|
||||||
for (int j = 0; j < item.Tags.Count; j++)
|
for (var j = 0; j < item.Tags.Count; j++) writer.WriteString(j.ToString(), item.Tags[j]);
|
||||||
{
|
writer.EndArray(tagsPos);
|
||||||
writer.WriteString(j.ToString(), item.Tags[j]);
|
|
||||||
}
|
|
||||||
writer.EndArray(tagsPos);
|
|
||||||
|
|
||||||
writer.EndDocument(itemPos);
|
writer.EndDocument(itemPos);
|
||||||
}
|
}
|
||||||
writer.EndArray(histPos);
|
|
||||||
|
writer.EndArray(histPos);
|
||||||
writer.EndDocument(sizePos);
|
|
||||||
|
writer.EndDocument(sizePos);
|
||||||
|
|
||||||
return writer.Position;
|
return writer.Position;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public override Person Deserialize(BsonSpanReader reader)
|
public override Person Deserialize(BsonSpanReader reader)
|
||||||
{
|
{
|
||||||
var person = new Person();
|
var person = new Person();
|
||||||
|
|
||||||
reader.ReadDocumentSize();
|
reader.ReadDocumentSize();
|
||||||
|
|
||||||
while (reader.Remaining > 0)
|
while (reader.Remaining > 0)
|
||||||
{
|
{
|
||||||
var type = reader.ReadBsonType();
|
var type = reader.ReadBsonType();
|
||||||
if (type == BsonType.EndOfDocument)
|
if (type == BsonType.EndOfDocument)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var name = reader.ReadElementHeader();
|
string name = reader.ReadElementHeader();
|
||||||
|
|
||||||
switch (name)
|
switch (name)
|
||||||
{
|
{
|
||||||
case "_id": person.Id = reader.ReadObjectId(); break;
|
case "_id": person.Id = reader.ReadObjectId(); break;
|
||||||
case "firstname": person.FirstName = reader.ReadString(); break;
|
case "firstname": person.FirstName = reader.ReadString(); break;
|
||||||
case "lastname": person.LastName = reader.ReadString(); break;
|
case "lastname": person.LastName = reader.ReadString(); break;
|
||||||
case "age": person.Age = reader.ReadInt32(); break;
|
case "age": person.Age = reader.ReadInt32(); break;
|
||||||
case "bio":
|
case "bio":
|
||||||
if (type == BsonType.Null) person.Bio = null;
|
if (type == BsonType.Null) person.Bio = null;
|
||||||
else person.Bio = reader.ReadString();
|
else person.Bio = reader.ReadString();
|
||||||
break;
|
break;
|
||||||
case "createdat": person.CreatedAt = new DateTime(reader.ReadInt64()); break;
|
case "createdat": person.CreatedAt = new DateTime(reader.ReadInt64()); break;
|
||||||
case "balance": person.Balance = (decimal)reader.ReadDouble(); break;
|
case "balance": person.Balance = (decimal)reader.ReadDouble(); break;
|
||||||
|
|
||||||
case "homeaddress":
|
case "homeaddress":
|
||||||
reader.ReadDocumentSize(); // Enter document
|
reader.ReadDocumentSize(); // Enter document
|
||||||
while (reader.Remaining > 0)
|
while (reader.Remaining > 0)
|
||||||
{
|
{
|
||||||
var addrType = reader.ReadBsonType();
|
var addrType = reader.ReadBsonType();
|
||||||
if (addrType == BsonType.EndOfDocument) break;
|
if (addrType == BsonType.EndOfDocument) break;
|
||||||
var addrName = reader.ReadElementHeader();
|
string addrName = reader.ReadElementHeader();
|
||||||
|
|
||||||
// We assume strict schema for benchmark speed, but should handle skipping
|
// We assume strict schema for benchmark speed, but should handle skipping
|
||||||
if (addrName == "street") person.HomeAddress.Street = reader.ReadString();
|
if (addrName == "street") person.HomeAddress.Street = reader.ReadString();
|
||||||
else if (addrName == "city") person.HomeAddress.City = reader.ReadString();
|
else if (addrName == "city") person.HomeAddress.City = reader.ReadString();
|
||||||
else if (addrName == "zipcode") person.HomeAddress.ZipCode = reader.ReadString();
|
else if (addrName == "zipcode") person.HomeAddress.ZipCode = reader.ReadString();
|
||||||
else reader.SkipValue(addrType);
|
else reader.SkipValue(addrType);
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
|
break;
|
||||||
|
|
||||||
case "employmenthistory":
|
case "employmenthistory":
|
||||||
reader.ReadDocumentSize(); // Enter Array
|
reader.ReadDocumentSize(); // Enter Array
|
||||||
while (reader.Remaining > 0)
|
while (reader.Remaining > 0)
|
||||||
{
|
{
|
||||||
var arrType = reader.ReadBsonType();
|
var arrType = reader.ReadBsonType();
|
||||||
if (arrType == BsonType.EndOfDocument) break;
|
if (arrType == BsonType.EndOfDocument) break;
|
||||||
reader.ReadElementHeader(); // Array index "0", "1"... ignore
|
reader.ReadElementHeader(); // Array index "0", "1"... ignore
|
||||||
|
|
||||||
// Read WorkHistory item
|
// Read WorkHistory item
|
||||||
var workItem = new WorkHistory();
|
var workItem = new WorkHistory();
|
||||||
reader.ReadDocumentSize(); // Enter Item Document
|
reader.ReadDocumentSize(); // Enter Item Document
|
||||||
while (reader.Remaining > 0)
|
while (reader.Remaining > 0)
|
||||||
{
|
{
|
||||||
var itemType = reader.ReadBsonType();
|
var itemType = reader.ReadBsonType();
|
||||||
if (itemType == BsonType.EndOfDocument) break;
|
if (itemType == BsonType.EndOfDocument) break;
|
||||||
var itemName = reader.ReadElementHeader();
|
string itemName = reader.ReadElementHeader();
|
||||||
|
|
||||||
if (itemName == "companyname") workItem.CompanyName = reader.ReadString();
|
if (itemName == "companyname")
|
||||||
else if (itemName == "title") workItem.Title = reader.ReadString();
|
{
|
||||||
else if (itemName == "durationyears") workItem.DurationYears = reader.ReadInt32();
|
workItem.CompanyName = reader.ReadString();
|
||||||
|
}
|
||||||
|
else if (itemName == "title")
|
||||||
|
{
|
||||||
|
workItem.Title = reader.ReadString();
|
||||||
|
}
|
||||||
|
else if (itemName == "durationyears")
|
||||||
|
{
|
||||||
|
workItem.DurationYears = reader.ReadInt32();
|
||||||
|
}
|
||||||
else if (itemName == "tags")
|
else if (itemName == "tags")
|
||||||
{
|
{
|
||||||
reader.ReadDocumentSize(); // Enter Tags Array
|
reader.ReadDocumentSize(); // Enter Tags Array
|
||||||
@@ -149,18 +161,23 @@ public class PersonMapper : ObjectIdMapperBase<Person>
|
|||||||
reader.SkipValue(tagType);
|
reader.SkipValue(tagType);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else reader.SkipValue(itemType);
|
else
|
||||||
|
{
|
||||||
|
reader.SkipValue(itemType);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
person.EmploymentHistory.Add(workItem);
|
person.EmploymentHistory.Add(workItem);
|
||||||
}
|
}
|
||||||
break;
|
|
||||||
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
reader.SkipValue(type);
|
reader.SkipValue(type);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return person;
|
return person;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user