Implement in-process multi-dataset sync isolation across core, network, persistence, and tests
All checks were successful
NuGet Package Publish / nuget (push) Successful in 1m14s
All checks were successful
NuGet Package Publish / nuget (push) Successful in 1m14s
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using ZB.MOM.WW.CBDDC.Core;
|
||||
using ZB.MOM.WW.CBDDC.Core.Network;
|
||||
using SurrealDb.Net;
|
||||
using ZB.MOM.WW.CBDDC.Core.Storage;
|
||||
@@ -46,6 +47,62 @@ public static class CBDDCSurrealEmbeddedExtensions
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers dataset synchronization options for a Surreal-backed dataset pipeline.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="configure">Optional per-dataset option overrides.</param>
|
||||
/// <returns>The service collection for chaining.</returns>
|
||||
public static IServiceCollection AddCBDDCSurrealEmbeddedDataset(
|
||||
this IServiceCollection services,
|
||||
string datasetId,
|
||||
Action<DatasetSyncOptions>? configure = null)
|
||||
{
|
||||
if (services == null) throw new ArgumentNullException(nameof(services));
|
||||
|
||||
var options = new DatasetSyncOptions
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(datasetId),
|
||||
Enabled = true
|
||||
};
|
||||
configure?.Invoke(options);
|
||||
options.DatasetId = DatasetId.Normalize(options.DatasetId);
|
||||
|
||||
services.AddSingleton(options);
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers dataset synchronization options for a Surreal-backed dataset pipeline.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <param name="configure">Configuration delegate for dataset options.</param>
|
||||
/// <returns>The service collection for chaining.</returns>
|
||||
public static IServiceCollection AddCBDDCSurrealEmbeddedDataset(
|
||||
this IServiceCollection services,
|
||||
Action<DatasetSyncOptions> configure)
|
||||
{
|
||||
if (configure == null) throw new ArgumentNullException(nameof(configure));
|
||||
|
||||
var options = new DatasetSyncOptions
|
||||
{
|
||||
DatasetId = DatasetId.Primary,
|
||||
Enabled = true
|
||||
};
|
||||
configure(options);
|
||||
|
||||
return services.AddCBDDCSurrealEmbeddedDataset(options.DatasetId, configured =>
|
||||
{
|
||||
configured.Enabled = options.Enabled;
|
||||
configured.SyncLoopDelay = options.SyncLoopDelay;
|
||||
configured.MaxPeersPerCycle = options.MaxPeersPerCycle;
|
||||
configured.MaxEntriesPerCycle = options.MaxEntriesPerCycle;
|
||||
configured.MaintenanceIntervalOverride = options.MaintenanceIntervalOverride;
|
||||
configured.InterestingCollections = options.InterestingCollections.ToList();
|
||||
});
|
||||
}
|
||||
|
||||
private static void RegisterCoreServices(
|
||||
IServiceCollection services,
|
||||
Func<IServiceProvider, CBDDCSurrealEmbeddedOptions> optionsFactory)
|
||||
|
||||
@@ -67,31 +67,31 @@ public sealed class CBDDCSurrealSchemaInitializer : ICBDDCSurrealSchemaInitializ
|
||||
{
|
||||
return $"""
|
||||
DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.OplogEntriesTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral};
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHashIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS hash UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS timestampPhysicalTime, timestampLogicalCounter, timestampNodeId;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS collection;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHashIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, hash UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, timestampPhysicalTime, timestampLogicalCounter, timestampNodeId;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.OplogCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.OplogEntriesTable} COLUMNS datasetId, collection;
|
||||
|
||||
DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral};
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS nodeId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS timestampPhysicalTime, timestampLogicalCounter;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS datasetId, nodeId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.SnapshotHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.SnapshotMetadataTable} COLUMNS datasetId, timestampPhysicalTime, timestampLogicalCounter;
|
||||
|
||||
DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral};
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerNodeIdIndex} ON TABLE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} COLUMNS nodeId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerEnabledIndex} ON TABLE {CBDDCSurrealSchemaNames.RemotePeerConfigurationsTable} COLUMNS isEnabled;
|
||||
|
||||
DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral};
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionKeyIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS collection, key UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS hlcPhysicalTime, hlcLogicalCounter, hlcNodeId;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS collection;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionKeyIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, collection, key UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, hlcPhysicalTime, hlcLogicalCounter, hlcNodeId;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.DocumentMetadataCollectionIndex} ON TABLE {CBDDCSurrealSchemaNames.DocumentMetadataTable} COLUMNS datasetId, collection;
|
||||
|
||||
DEFINE TABLE OVERWRITE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} SCHEMAFULL CHANGEFEED {_changefeedRetentionLiteral};
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationPairIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS peerNodeId, sourceNodeId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationActiveIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS isActive;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationSourceHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS sourceNodeId, confirmedWall, confirmedLogic;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationPairIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, peerNodeId, sourceNodeId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationActiveIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, isActive;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.PeerConfirmationSourceHlcIndex} ON TABLE {CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable} COLUMNS datasetId, sourceNodeId, confirmedWall, confirmedLogic;
|
||||
|
||||
DEFINE TABLE OVERWRITE {_checkpointTable} SCHEMAFULL;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointConsumerIndex} ON TABLE {_checkpointTable} COLUMNS consumerId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointVersionstampIndex} ON TABLE {_checkpointTable} COLUMNS versionstampCursor;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointConsumerIndex} ON TABLE {_checkpointTable} COLUMNS datasetId, consumerId UNIQUE;
|
||||
DEFINE INDEX OVERWRITE {CBDDCSurrealSchemaNames.CdcCheckpointVersionstampIndex} ON TABLE {_checkpointTable} COLUMNS datasetId, versionstampCursor;
|
||||
""";
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,11 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal;
|
||||
/// </summary>
|
||||
public sealed class SurrealCdcCheckpoint
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the logical consumer identifier.
|
||||
/// </summary>
|
||||
@@ -48,6 +53,21 @@ public interface ISurrealCdcCheckpointPersistence
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Reads the checkpoint for a consumer within a specific dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="consumerId">Optional consumer id.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>The checkpoint if found; otherwise <see langword="null" />.</returns>
|
||||
Task<SurrealCdcCheckpoint?> GetCheckpointAsync(
|
||||
string datasetId,
|
||||
string? consumerId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return GetCheckpointAsync(consumerId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Upserts checkpoint progress for a consumer.
|
||||
/// </summary>
|
||||
@@ -63,6 +83,26 @@ public interface ISurrealCdcCheckpointPersistence
|
||||
CancellationToken cancellationToken = default,
|
||||
long? versionstampCursor = null);
|
||||
|
||||
/// <summary>
|
||||
/// Upserts checkpoint progress for a consumer and dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="timestamp">The last processed timestamp.</param>
|
||||
/// <param name="lastHash">The last processed hash.</param>
|
||||
/// <param name="consumerId">Optional consumer id.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <param name="versionstampCursor">Optional changefeed versionstamp cursor.</param>
|
||||
Task UpsertCheckpointAsync(
|
||||
string datasetId,
|
||||
HlcTimestamp timestamp,
|
||||
string lastHash,
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
long? versionstampCursor = null)
|
||||
{
|
||||
return UpsertCheckpointAsync(timestamp, lastHash, consumerId, cancellationToken, versionstampCursor);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Advances checkpoint progress from an oplog entry.
|
||||
/// </summary>
|
||||
@@ -73,4 +113,20 @@ public interface ISurrealCdcCheckpointPersistence
|
||||
OplogEntry entry,
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Advances checkpoint progress for a specific dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="entry">The oplog entry that was processed.</param>
|
||||
/// <param name="consumerId">Optional consumer id.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
Task AdvanceCheckpointAsync(
|
||||
string datasetId,
|
||||
OplogEntry entry,
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return AdvanceCheckpointAsync(entry, consumerId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,11 +49,21 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi
|
||||
public async Task<SurrealCdcCheckpoint?> GetCheckpointAsync(
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await GetCheckpointAsync(DatasetId.Primary, consumerId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<SurrealCdcCheckpoint?> GetCheckpointAsync(
|
||||
string datasetId,
|
||||
string? consumerId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (!_enabled) return null;
|
||||
|
||||
string resolvedConsumerId = ResolveConsumerId(consumerId);
|
||||
var existing = await FindByConsumerIdAsync(resolvedConsumerId, cancellationToken);
|
||||
string resolvedDatasetId = DatasetId.Normalize(datasetId);
|
||||
var existing = await FindByConsumerIdAsync(resolvedDatasetId, resolvedConsumerId, cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
@@ -64,26 +74,47 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
long? versionstampCursor = null)
|
||||
{
|
||||
await UpsertCheckpointAsync(
|
||||
DatasetId.Primary,
|
||||
timestamp,
|
||||
lastHash,
|
||||
consumerId,
|
||||
cancellationToken,
|
||||
versionstampCursor);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertCheckpointAsync(
|
||||
string datasetId,
|
||||
HlcTimestamp timestamp,
|
||||
string lastHash,
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default,
|
||||
long? versionstampCursor = null)
|
||||
{
|
||||
if (!_enabled) return;
|
||||
|
||||
string resolvedConsumerId = ResolveConsumerId(consumerId);
|
||||
string resolvedDatasetId = DatasetId.Normalize(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
|
||||
long? effectiveVersionstampCursor = versionstampCursor;
|
||||
if (!effectiveVersionstampCursor.HasValue)
|
||||
{
|
||||
var existing = await FindByConsumerIdAsync(
|
||||
resolvedDatasetId,
|
||||
resolvedConsumerId,
|
||||
cancellationToken,
|
||||
ensureInitialized: false);
|
||||
effectiveVersionstampCursor = existing?.VersionstampCursor;
|
||||
}
|
||||
|
||||
RecordId recordId = RecordId.From(_checkpointTable, ComputeConsumerKey(resolvedConsumerId));
|
||||
RecordId recordId = RecordId.From(_checkpointTable, ComputeConsumerKey(resolvedDatasetId, resolvedConsumerId));
|
||||
|
||||
var record = new SurrealCdcCheckpointRecord
|
||||
{
|
||||
DatasetId = resolvedDatasetId,
|
||||
ConsumerId = resolvedConsumerId,
|
||||
TimestampPhysicalTime = timestamp.PhysicalTime,
|
||||
TimestampLogicalCounter = timestamp.LogicalCounter,
|
||||
@@ -106,7 +137,18 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
return UpsertCheckpointAsync(entry.Timestamp, entry.Hash, consumerId, cancellationToken);
|
||||
return UpsertCheckpointAsync(entry.DatasetId, entry.Timestamp, entry.Hash, consumerId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task AdvanceCheckpointAsync(
|
||||
string datasetId,
|
||||
OplogEntry entry,
|
||||
string? consumerId = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(entry);
|
||||
return UpsertCheckpointAsync(datasetId, entry.Timestamp, entry.Hash, consumerId, cancellationToken);
|
||||
}
|
||||
|
||||
private string ResolveConsumerId(string? consumerId)
|
||||
@@ -124,32 +166,44 @@ public sealed class SurrealCdcCheckpointPersistence : ISurrealCdcCheckpointPersi
|
||||
}
|
||||
|
||||
private async Task<SurrealCdcCheckpointRecord?> FindByConsumerIdAsync(
|
||||
string datasetId,
|
||||
string consumerId,
|
||||
CancellationToken cancellationToken,
|
||||
bool ensureInitialized = true)
|
||||
{
|
||||
string normalizedDatasetId = DatasetId.Normalize(datasetId);
|
||||
if (ensureInitialized) await EnsureReadyAsync(cancellationToken);
|
||||
|
||||
RecordId deterministicId = RecordId.From(_checkpointTable, ComputeConsumerKey(consumerId));
|
||||
RecordId deterministicId = RecordId.From(_checkpointTable, ComputeConsumerKey(normalizedDatasetId, consumerId));
|
||||
var deterministic = await _surrealClient.Select<SurrealCdcCheckpointRecord>(deterministicId, cancellationToken);
|
||||
if (deterministic != null &&
|
||||
string.Equals(deterministic.DatasetId, normalizedDatasetId, StringComparison.Ordinal) &&
|
||||
string.Equals(deterministic.ConsumerId, consumerId, StringComparison.Ordinal))
|
||||
return deterministic;
|
||||
|
||||
var all = await _surrealClient.Select<SurrealCdcCheckpointRecord>(_checkpointTable, cancellationToken);
|
||||
return all?.FirstOrDefault(c =>
|
||||
(string.IsNullOrWhiteSpace(c.DatasetId)
|
||||
? string.Equals(normalizedDatasetId, DatasetId.Primary, StringComparison.Ordinal)
|
||||
: string.Equals(c.DatasetId, normalizedDatasetId, StringComparison.Ordinal)) &&
|
||||
string.Equals(c.ConsumerId, consumerId, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private static string ComputeConsumerKey(string consumerId)
|
||||
private static string ComputeConsumerKey(string datasetId, string consumerId)
|
||||
{
|
||||
byte[] input = Encoding.UTF8.GetBytes(consumerId);
|
||||
byte[] input = Encoding.UTF8.GetBytes($"{datasetId}\n{consumerId}");
|
||||
return Convert.ToHexString(SHA256.HashData(input)).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class SurrealCdcCheckpointRecord : Record
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("datasetId")]
|
||||
public string DatasetId { get; set; } = global::ZB.MOM.WW.CBDDC.Core.DatasetId.Primary;
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the CDC consumer identifier.
|
||||
/// </summary>
|
||||
@@ -204,6 +258,7 @@ internal static class SurrealCdcCheckpointRecordMappers
|
||||
{
|
||||
return new SurrealCdcCheckpoint
|
||||
{
|
||||
DatasetId = string.IsNullOrWhiteSpace(record.DatasetId) ? DatasetId.Primary : record.DatasetId,
|
||||
ConsumerId = record.ConsumerId,
|
||||
Timestamp = new HlcTimestamp(
|
||||
record.TimestampPhysicalTime,
|
||||
|
||||
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal;
|
||||
|
||||
public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
{
|
||||
private const string PrimaryDatasetId = DatasetId.Primary;
|
||||
private readonly ILogger<SurrealDocumentMetadataStore> _logger;
|
||||
private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer;
|
||||
private readonly ISurrealDbClient _surrealClient;
|
||||
@@ -30,11 +31,35 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
_logger = logger ?? NullLogger<SurrealDocumentMetadataStore>.Instance;
|
||||
}
|
||||
|
||||
private static string NormalizeDatasetId(string? datasetId)
|
||||
{
|
||||
return DatasetId.Normalize(datasetId);
|
||||
}
|
||||
|
||||
private static bool MatchesDataset(string? recordDatasetId, string datasetId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(recordDatasetId))
|
||||
return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal);
|
||||
|
||||
return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<DocumentMetadata?> GetMetadataAsync(string collection, string key,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByCollectionKeyAsync(collection, key, cancellationToken);
|
||||
var existing = await FindByCollectionKeyAsync(collection, key, PrimaryDatasetId, cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DocumentMetadata?> GetMetadataAsync(
|
||||
string collection,
|
||||
string key,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByCollectionKeyAsync(collection, key, NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
@@ -42,7 +67,20 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
public override async Task<IEnumerable<DocumentMetadata>> GetMetadataByCollectionAsync(string collection,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
return all
|
||||
.Where(m => string.Equals(m.Collection, collection, StringComparison.Ordinal))
|
||||
.Select(m => m.ToDomain())
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<DocumentMetadata>> GetMetadataByCollectionAsync(
|
||||
string collection,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all
|
||||
.Where(m => string.Equals(m.Collection, collection, StringComparison.Ordinal))
|
||||
.Select(m => m.ToDomain())
|
||||
@@ -53,10 +91,26 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
public override async Task UpsertMetadataAsync(DocumentMetadata metadata,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await UpsertMetadataAsync(metadata, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertMetadataAsync(
|
||||
DocumentMetadata metadata,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
metadata.DatasetId = normalizedDatasetId;
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
|
||||
var existing = await FindByCollectionKeyAsync(metadata.Collection, metadata.Key, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.DocumentMetadata(metadata.Collection, metadata.Key);
|
||||
var existing =
|
||||
await FindByCollectionKeyAsync(metadata.Collection, metadata.Key, normalizedDatasetId, cancellationToken);
|
||||
RecordId recordId = existing?.Id ??
|
||||
SurrealStoreRecordIds.DocumentMetadata(
|
||||
metadata.Collection,
|
||||
metadata.Key,
|
||||
normalizedDatasetId);
|
||||
|
||||
await _surrealClient.Upsert<SurrealDocumentMetadataRecord, SurrealDocumentMetadataRecord>(
|
||||
recordId,
|
||||
@@ -67,24 +121,46 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
/// <inheritdoc />
|
||||
public override async Task UpsertMetadataBatchAsync(IEnumerable<DocumentMetadata> metadatas,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await UpsertMetadataBatchAsync(metadatas, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertMetadataBatchAsync(IEnumerable<DocumentMetadata> metadatas,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var metadata in metadatas)
|
||||
await UpsertMetadataAsync(metadata, cancellationToken);
|
||||
await UpsertMetadataAsync(metadata, datasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var metadata = new DocumentMetadata(collection, key, timestamp, true);
|
||||
await UpsertMetadataAsync(metadata, cancellationToken);
|
||||
await MarkDeletedAsync(collection, key, timestamp, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task MarkDeletedAsync(string collection, string key, HlcTimestamp timestamp, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var metadata = new DocumentMetadata(collection, key, timestamp, true, datasetId);
|
||||
await UpsertMetadataAsync(metadata, datasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<DocumentMetadata>> GetMetadataAfterAsync(HlcTimestamp since,
|
||||
IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
return await GetMetadataAfterAsync(since, PrimaryDatasetId, collections, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<DocumentMetadata>> GetMetadataAfterAsync(HlcTimestamp since, string datasetId,
|
||||
IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
HashSet<string>? collectionSet = collections != null ? new HashSet<string>(collections) : null;
|
||||
|
||||
return all
|
||||
@@ -101,14 +177,45 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
/// <inheritdoc />
|
||||
public override async Task DropAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient.Delete(CBDDCSurrealSchemaNames.DocumentMetadataTable, cancellationToken);
|
||||
await DropAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops all metadata rows for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var recordId = row.Id ??
|
||||
SurrealStoreRecordIds.DocumentMetadata(
|
||||
row.Collection,
|
||||
row.Key,
|
||||
NormalizeDatasetId(datasetId));
|
||||
await _surrealClient.Delete(recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<DocumentMetadata>> ExportAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
return all.Select(m => m.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports metadata rows for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>The exported metadata rows.</returns>
|
||||
public async Task<IEnumerable<DocumentMetadata>> ExportAsync(string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all.Select(m => m.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
@@ -116,16 +223,42 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
public override async Task ImportAsync(IEnumerable<DocumentMetadata> items,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var item in items) await UpsertMetadataAsync(item, cancellationToken);
|
||||
await ImportAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports metadata rows into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The metadata items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task ImportAsync(IEnumerable<DocumentMetadata> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
foreach (var item in items) await UpsertMetadataAsync(item, datasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task MergeAsync(IEnumerable<DocumentMetadata> items,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await MergeAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges metadata rows into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The metadata items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task MergeAsync(IEnumerable<DocumentMetadata> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByCollectionKeyAsync(item.Collection, item.Key, cancellationToken);
|
||||
item.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByCollectionKeyAsync(item.Collection, item.Key, normalizedDatasetId, cancellationToken);
|
||||
|
||||
if (existing == null)
|
||||
{
|
||||
@@ -138,7 +271,8 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
|
||||
if (item.UpdatedAt.CompareTo(existingTimestamp) <= 0) continue;
|
||||
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.DocumentMetadata(item.Collection, item.Key);
|
||||
RecordId recordId =
|
||||
existing.Id ?? SurrealStoreRecordIds.DocumentMetadata(item.Collection, item.Key, normalizedDatasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient.Upsert<SurrealDocumentMetadataRecord, SurrealDocumentMetadataRecord>(
|
||||
recordId,
|
||||
@@ -152,27 +286,37 @@ public class SurrealDocumentMetadataStore : DocumentMetadataStore
|
||||
await _schemaInitializer.EnsureInitializedAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<List<SurrealDocumentMetadataRecord>> SelectAllAsync(CancellationToken cancellationToken)
|
||||
private async Task<List<SurrealDocumentMetadataRecord>> SelectAllAsync(string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
var rows = await _surrealClient.Select<SurrealDocumentMetadataRecord>(
|
||||
CBDDCSurrealSchemaNames.DocumentMetadataTable,
|
||||
cancellationToken);
|
||||
return rows?.ToList() ?? [];
|
||||
return rows?
|
||||
.Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId))
|
||||
.ToList()
|
||||
?? [];
|
||||
}
|
||||
|
||||
private async Task<SurrealDocumentMetadataRecord?> FindByCollectionKeyAsync(string collection, string key,
|
||||
private async Task<SurrealDocumentMetadataRecord?> FindByCollectionKeyAsync(
|
||||
string collection,
|
||||
string key,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.DocumentMetadata(collection, key);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.DocumentMetadata(collection, key, normalizedDatasetId);
|
||||
var deterministic = await _surrealClient.Select<SurrealDocumentMetadataRecord>(deterministicId, cancellationToken);
|
||||
if (deterministic != null &&
|
||||
MatchesDataset(deterministic.DatasetId, normalizedDatasetId) &&
|
||||
string.Equals(deterministic.Collection, collection, StringComparison.Ordinal) &&
|
||||
string.Equals(deterministic.Key, key, StringComparison.Ordinal))
|
||||
return deterministic;
|
||||
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
return all.FirstOrDefault(m =>
|
||||
string.Equals(m.Collection, collection, StringComparison.Ordinal) &&
|
||||
string.Equals(m.Key, key, StringComparison.Ordinal));
|
||||
|
||||
@@ -1205,7 +1205,10 @@ public abstract class SurrealDocumentStore<TContext> : IDocumentStore, ISurrealC
|
||||
{
|
||||
["oplogRecordId"] = SurrealStoreRecordIds.Oplog(oplogEntry.Hash),
|
||||
["oplogRecord"] = oplogEntry.ToSurrealRecord(),
|
||||
["metadataRecordId"] = SurrealStoreRecordIds.DocumentMetadata(metadata.Collection, metadata.Key),
|
||||
["metadataRecordId"] = SurrealStoreRecordIds.DocumentMetadata(
|
||||
metadata.Collection,
|
||||
metadata.Key,
|
||||
metadata.DatasetId),
|
||||
["metadataRecord"] = metadata.ToSurrealRecord()
|
||||
};
|
||||
|
||||
@@ -1261,10 +1264,12 @@ public abstract class SurrealDocumentStore<TContext> : IDocumentStore, ISurrealC
|
||||
checkpointRecord = new Dictionary<string, object?>();
|
||||
if (!TryGetCheckpointSettings(out string checkpointTable, out string consumerId)) return false;
|
||||
|
||||
string consumerKey = ComputeConsumerKey(consumerId);
|
||||
const string datasetId = DatasetId.Primary;
|
||||
string consumerKey = ComputeConsumerKey(datasetId, consumerId);
|
||||
checkpointRecordId = RecordId.From(checkpointTable, consumerKey);
|
||||
checkpointRecord = new Dictionary<string, object?>
|
||||
{
|
||||
["datasetId"] = datasetId,
|
||||
["consumerId"] = consumerId,
|
||||
["timestampPhysicalTime"] = oplogEntry.Timestamp.PhysicalTime,
|
||||
["timestampLogicalCounter"] = oplogEntry.Timestamp.LogicalCounter,
|
||||
@@ -1294,10 +1299,12 @@ public abstract class SurrealDocumentStore<TContext> : IDocumentStore, ISurrealC
|
||||
? long.MaxValue
|
||||
: (long)pendingCursorCheckpoint.Value.Cursor;
|
||||
|
||||
string consumerKey = ComputeConsumerKey(cursorConsumerId);
|
||||
const string datasetId = DatasetId.Primary;
|
||||
string consumerKey = ComputeConsumerKey(datasetId, cursorConsumerId);
|
||||
checkpointRecordId = RecordId.From(checkpointTable, consumerKey);
|
||||
checkpointRecord = new Dictionary<string, object?>
|
||||
{
|
||||
["datasetId"] = datasetId,
|
||||
["consumerId"] = cursorConsumerId,
|
||||
["timestampPhysicalTime"] = encodedCursor,
|
||||
["timestampLogicalCounter"] = 0,
|
||||
@@ -1329,9 +1336,9 @@ public abstract class SurrealDocumentStore<TContext> : IDocumentStore, ISurrealC
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string ComputeConsumerKey(string consumerId)
|
||||
private static string ComputeConsumerKey(string datasetId, string consumerId)
|
||||
{
|
||||
byte[] bytes = Encoding.UTF8.GetBytes(consumerId);
|
||||
byte[] bytes = Encoding.UTF8.GetBytes($"{datasetId}\n{consumerId}");
|
||||
return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal;
|
||||
|
||||
public class SurrealOplogStore : OplogStore
|
||||
{
|
||||
private const string PrimaryDatasetId = DatasetId.Primary;
|
||||
private readonly ILogger<SurrealOplogStore> _logger;
|
||||
private readonly ICBDDCSurrealSchemaInitializer? _schemaInitializer;
|
||||
private readonly ISurrealDbClient? _surrealClient;
|
||||
@@ -46,17 +47,38 @@ public class SurrealOplogStore : OplogStore
|
||||
InitializeVectorClock();
|
||||
}
|
||||
|
||||
private static string NormalizeDatasetId(string? datasetId)
|
||||
{
|
||||
return DatasetId.Normalize(datasetId);
|
||||
}
|
||||
|
||||
private static bool MatchesDataset(string? recordDatasetId, string datasetId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(recordDatasetId))
|
||||
return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal);
|
||||
|
||||
return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<OplogEntry>> GetChainRangeAsync(string startHash, string endHash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var startRow = await FindByHashAsync(startHash, cancellationToken);
|
||||
var endRow = await FindByHashAsync(endHash, cancellationToken);
|
||||
return await GetChainRangeAsync(startHash, endHash, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<OplogEntry>> GetChainRangeAsync(string startHash, string endHash, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var startRow = await FindByHashAsync(startHash, normalizedDatasetId, cancellationToken);
|
||||
var endRow = await FindByHashAsync(endHash, normalizedDatasetId, cancellationToken);
|
||||
|
||||
if (startRow == null || endRow == null) return [];
|
||||
|
||||
string nodeId = startRow.TimestampNodeId;
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
|
||||
return all
|
||||
.Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal) &&
|
||||
@@ -75,7 +97,16 @@ public class SurrealOplogStore : OplogStore
|
||||
/// <inheritdoc />
|
||||
public override async Task<OplogEntry?> GetEntryByHashAsync(string hash, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByHashAsync(hash, cancellationToken);
|
||||
var existing = await FindByHashAsync(hash, PrimaryDatasetId, cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<OplogEntry?> GetEntryByHashAsync(string hash, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var existing = await FindByHashAsync(hash, normalizedDatasetId, cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
@@ -83,7 +114,15 @@ public class SurrealOplogStore : OplogStore
|
||||
public override async Task<IEnumerable<OplogEntry>> GetOplogAfterAsync(HlcTimestamp timestamp,
|
||||
IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
return await GetOplogAfterAsync(timestamp, PrimaryDatasetId, collections, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<OplogEntry>> GetOplogAfterAsync(HlcTimestamp timestamp, string datasetId,
|
||||
IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
HashSet<string>? collectionSet = collections != null ? new HashSet<string>(collections) : null;
|
||||
|
||||
return all
|
||||
@@ -102,7 +141,15 @@ public class SurrealOplogStore : OplogStore
|
||||
public override async Task<IEnumerable<OplogEntry>> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since,
|
||||
IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
return await GetOplogForNodeAfterAsync(nodeId, since, PrimaryDatasetId, collections, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<OplogEntry>> GetOplogForNodeAfterAsync(string nodeId, HlcTimestamp since,
|
||||
string datasetId, IEnumerable<string>? collections = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
HashSet<string>? collectionSet = collections != null ? new HashSet<string>(collections) : null;
|
||||
|
||||
return all
|
||||
@@ -121,7 +168,15 @@ public class SurrealOplogStore : OplogStore
|
||||
/// <inheritdoc />
|
||||
public override async Task PruneOplogAsync(HlcTimestamp cutoff, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
await PruneOplogAsync(cutoff, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task PruneOplogAsync(HlcTimestamp cutoff, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
var toDelete = all
|
||||
.Where(o => o.TimestampPhysicalTime < cutoff.PhysicalTime ||
|
||||
(o.TimestampPhysicalTime == cutoff.PhysicalTime &&
|
||||
@@ -139,38 +194,114 @@ public class SurrealOplogStore : OplogStore
|
||||
/// <inheritdoc />
|
||||
public override async Task DropAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient!.Delete(CBDDCSurrealSchemaNames.OplogEntriesTable, cancellationToken);
|
||||
await DropAsync(PrimaryDatasetId, cancellationToken);
|
||||
_vectorClock.Invalidate();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops all oplog rows for the specified dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
RecordId recordId = row.Id ?? SurrealStoreRecordIds.Oplog(row.Hash);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient!.Delete(recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<OplogEntry>> ExportAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
return all.Select(o => o.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports all oplog entries for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>All dataset-scoped oplog entries.</returns>
|
||||
public async Task<IEnumerable<OplogEntry>> ExportAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all.Select(o => o.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task ImportAsync(IEnumerable<OplogEntry> items, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await ImportAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports oplog entries for the specified dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The entries to import.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task ImportAsync(IEnumerable<OplogEntry> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
Dictionary<string, RecordId> existingByHash =
|
||||
await LoadOplogRecordIdsByHashAsync(normalizedDatasetId, cancellationToken);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByHashAsync(item.Hash, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.Oplog(item.Hash);
|
||||
await UpsertAsync(item, recordId, cancellationToken);
|
||||
var normalizedItem = new OplogEntry(
|
||||
item.Collection,
|
||||
item.Key,
|
||||
item.Operation,
|
||||
item.Payload,
|
||||
item.Timestamp,
|
||||
item.PreviousHash,
|
||||
item.Hash,
|
||||
normalizedDatasetId);
|
||||
RecordId recordId = existingByHash.TryGetValue(item.Hash, out RecordId? existingRecordId)
|
||||
? existingRecordId
|
||||
: SurrealStoreRecordIds.Oplog(item.Hash);
|
||||
await UpsertAsync(normalizedItem, recordId, cancellationToken);
|
||||
existingByHash[item.Hash] = recordId;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task MergeAsync(IEnumerable<OplogEntry> items, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await MergeAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges oplog entries into the specified dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The entries to merge.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task MergeAsync(IEnumerable<OplogEntry> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
HashSet<string> existingHashes = await LoadOplogHashesAsync(normalizedDatasetId, cancellationToken);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByHashAsync(item.Hash, cancellationToken);
|
||||
if (existing != null) continue;
|
||||
if (!existingHashes.Add(item.Hash))
|
||||
continue;
|
||||
|
||||
await UpsertAsync(item, SurrealStoreRecordIds.Oplog(item.Hash), cancellationToken);
|
||||
var normalizedItem = new OplogEntry(
|
||||
item.Collection,
|
||||
item.Key,
|
||||
item.Operation,
|
||||
item.Payload,
|
||||
item.Timestamp,
|
||||
item.PreviousHash,
|
||||
item.Hash,
|
||||
normalizedDatasetId);
|
||||
await UpsertAsync(normalizedItem, SurrealStoreRecordIds.Oplog(normalizedItem.Hash), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,6 +321,8 @@ public class SurrealOplogStore : OplogStore
|
||||
{
|
||||
var snapshots = _snapshotMetadataStore.GetAllSnapshotMetadataAsync().GetAwaiter().GetResult();
|
||||
foreach (var snapshot in snapshots)
|
||||
{
|
||||
if (!MatchesDataset(snapshot.DatasetId, PrimaryDatasetId)) continue;
|
||||
_vectorClock.UpdateNode(
|
||||
snapshot.NodeId,
|
||||
new HlcTimestamp(
|
||||
@@ -197,6 +330,7 @@ public class SurrealOplogStore : OplogStore
|
||||
snapshot.TimestampLogicalCounter,
|
||||
snapshot.NodeId),
|
||||
snapshot.Hash ?? "");
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -209,6 +343,7 @@ public class SurrealOplogStore : OplogStore
|
||||
?? [];
|
||||
|
||||
var latestPerNode = all
|
||||
.Where(x => MatchesDataset(x.DatasetId, PrimaryDatasetId))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x.TimestampNodeId))
|
||||
.GroupBy(x => x.TimestampNodeId)
|
||||
.Select(g => g
|
||||
@@ -229,17 +364,27 @@ public class SurrealOplogStore : OplogStore
|
||||
/// <inheritdoc />
|
||||
protected override async Task InsertOplogEntryAsync(OplogEntry entry, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByHashAsync(entry.Hash, cancellationToken);
|
||||
string datasetId = NormalizeDatasetId(entry.DatasetId);
|
||||
var normalizedEntry = new OplogEntry(
|
||||
entry.Collection,
|
||||
entry.Key,
|
||||
entry.Operation,
|
||||
entry.Payload,
|
||||
entry.Timestamp,
|
||||
entry.PreviousHash,
|
||||
entry.Hash,
|
||||
datasetId);
|
||||
var existing = await FindByHashAsync(normalizedEntry.Hash, datasetId, cancellationToken);
|
||||
if (existing != null) return;
|
||||
|
||||
await UpsertAsync(entry, SurrealStoreRecordIds.Oplog(entry.Hash), cancellationToken);
|
||||
await UpsertAsync(normalizedEntry, SurrealStoreRecordIds.Oplog(normalizedEntry.Hash), cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task<string?> QueryLastHashForNodeAsync(string nodeId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
var lastEntry = all
|
||||
.Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal))
|
||||
.OrderByDescending(o => o.TimestampPhysicalTime)
|
||||
@@ -252,11 +397,106 @@ public class SurrealOplogStore : OplogStore
|
||||
protected override async Task<(long Wall, int Logic)?> QueryLastHashTimestampFromOplogAsync(string hash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByHashAsync(hash, cancellationToken);
|
||||
var existing = await FindByHashAsync(hash, PrimaryDatasetId, cancellationToken);
|
||||
if (existing == null) return null;
|
||||
return (existing.TimestampPhysicalTime, existing.TimestampLogicalCounter);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task AppendOplogEntryAsync(
|
||||
OplogEntry entry,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var normalizedEntry = new OplogEntry(
|
||||
entry.Collection,
|
||||
entry.Key,
|
||||
entry.Operation,
|
||||
entry.Payload,
|
||||
entry.Timestamp,
|
||||
entry.PreviousHash,
|
||||
entry.Hash,
|
||||
normalizedDatasetId);
|
||||
await AppendOplogEntryAsync(normalizedEntry, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<HlcTimestamp> GetLatestTimestampAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
var latest = all
|
||||
.OrderByDescending(o => o.TimestampPhysicalTime)
|
||||
.ThenByDescending(o => o.TimestampLogicalCounter)
|
||||
.FirstOrDefault();
|
||||
|
||||
return latest == null
|
||||
? new HlcTimestamp(0, 0, "")
|
||||
: new HlcTimestamp(latest.TimestampPhysicalTime, latest.TimestampLogicalCounter, latest.TimestampNodeId);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<VectorClock> GetVectorClockAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
var vectorClock = new VectorClock();
|
||||
foreach (var latest in all
|
||||
.Where(o => !string.IsNullOrWhiteSpace(o.TimestampNodeId))
|
||||
.GroupBy(o => o.TimestampNodeId)
|
||||
.Select(g => g.OrderByDescending(o => o.TimestampPhysicalTime)
|
||||
.ThenByDescending(o => o.TimestampLogicalCounter)
|
||||
.First()))
|
||||
vectorClock.SetTimestamp(
|
||||
latest.TimestampNodeId,
|
||||
new HlcTimestamp(latest.TimestampPhysicalTime, latest.TimestampLogicalCounter, latest.TimestampNodeId));
|
||||
|
||||
return vectorClock;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<string?> GetLastEntryHashAsync(string nodeId, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
var latest = all
|
||||
.Where(o => string.Equals(o.TimestampNodeId, nodeId, StringComparison.Ordinal))
|
||||
.OrderByDescending(o => o.TimestampPhysicalTime)
|
||||
.ThenByDescending(o => o.TimestampLogicalCounter)
|
||||
.FirstOrDefault();
|
||||
|
||||
if (latest != null) return latest.Hash;
|
||||
|
||||
if (_snapshotMetadataStore == null) return null;
|
||||
var snapshotHash =
|
||||
await _snapshotMetadataStore.GetSnapshotHashAsync(nodeId, normalizedDatasetId, cancellationToken);
|
||||
return snapshotHash;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task ApplyBatchAsync(
|
||||
IEnumerable<OplogEntry> oplogEntries,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
var normalizedEntries = oplogEntries
|
||||
.Select(entry => new OplogEntry(
|
||||
entry.Collection,
|
||||
entry.Key,
|
||||
entry.Operation,
|
||||
entry.Payload,
|
||||
entry.Timestamp,
|
||||
entry.PreviousHash,
|
||||
entry.Hash,
|
||||
normalizedDatasetId))
|
||||
.ToList();
|
||||
|
||||
await ApplyBatchAsync(normalizedEntries, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task UpsertAsync(OplogEntry entry, RecordId recordId, CancellationToken cancellationToken)
|
||||
{
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
@@ -271,25 +511,56 @@ public class SurrealOplogStore : OplogStore
|
||||
await _schemaInitializer!.EnsureInitializedAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<List<SurrealOplogRecord>> SelectAllAsync(CancellationToken cancellationToken)
|
||||
private async Task<List<SurrealOplogRecord>> SelectAllAsync(string datasetId, CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
var rows = await _surrealClient!.Select<SurrealOplogRecord>(
|
||||
CBDDCSurrealSchemaNames.OplogEntriesTable,
|
||||
cancellationToken);
|
||||
return rows?.ToList() ?? [];
|
||||
return rows?
|
||||
.Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId))
|
||||
.ToList()
|
||||
?? [];
|
||||
}
|
||||
|
||||
private async Task<SurrealOplogRecord?> FindByHashAsync(string hash, CancellationToken cancellationToken)
|
||||
private async Task<SurrealOplogRecord?> FindByHashAsync(string hash, string datasetId, CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
|
||||
RecordId deterministicId = SurrealStoreRecordIds.Oplog(hash);
|
||||
var deterministic = await _surrealClient!.Select<SurrealOplogRecord>(deterministicId, cancellationToken);
|
||||
if (deterministic != null && string.Equals(deterministic.Hash, hash, StringComparison.Ordinal))
|
||||
if (deterministic != null &&
|
||||
string.Equals(deterministic.Hash, hash, StringComparison.Ordinal) &&
|
||||
MatchesDataset(deterministic.DatasetId, normalizedDatasetId))
|
||||
return deterministic;
|
||||
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
return all.FirstOrDefault(o => string.Equals(o.Hash, hash, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private async Task<HashSet<string>> LoadOplogHashesAsync(string datasetId, CancellationToken cancellationToken)
|
||||
{
|
||||
var rows = await SelectAllAsync(datasetId, cancellationToken);
|
||||
return rows
|
||||
.Where(r => !string.IsNullOrWhiteSpace(r.Hash))
|
||||
.Select(r => r.Hash)
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private async Task<Dictionary<string, RecordId>> LoadOplogRecordIdsByHashAsync(string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var rows = await SelectAllAsync(datasetId, cancellationToken);
|
||||
var records = new Dictionary<string, RecordId>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(row.Hash)) continue;
|
||||
records[row.Hash] = row.Id ?? SurrealStoreRecordIds.Oplog(row.Hash);
|
||||
}
|
||||
|
||||
return records;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal;
|
||||
public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
{
|
||||
internal const string RegistrationSourceNodeId = "__peer_registration__";
|
||||
private const string PrimaryDatasetId = DatasetId.Primary;
|
||||
|
||||
private readonly ILogger<SurrealPeerOplogConfirmationStore> _logger;
|
||||
private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer;
|
||||
@@ -32,6 +33,19 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
_logger = logger ?? NullLogger<SurrealPeerOplogConfirmationStore>.Instance;
|
||||
}
|
||||
|
||||
private static string NormalizeDatasetId(string? datasetId)
|
||||
{
|
||||
return DatasetId.Normalize(datasetId);
|
||||
}
|
||||
|
||||
private static bool MatchesDataset(string? recordDatasetId, string datasetId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(recordDatasetId))
|
||||
return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal);
|
||||
|
||||
return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task EnsurePeerRegisteredAsync(
|
||||
string peerNodeId,
|
||||
@@ -39,16 +53,29 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
PeerType type,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsurePeerRegisteredAsync(peerNodeId, address, type, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task EnsurePeerRegisteredAsync(
|
||||
string peerNodeId,
|
||||
string address,
|
||||
PeerType type,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
if (string.IsNullOrWhiteSpace(peerNodeId))
|
||||
throw new ArgumentException("Peer node id is required.", nameof(peerNodeId));
|
||||
|
||||
var existing =
|
||||
await FindByPairAsync(peerNodeId, RegistrationSourceNodeId, cancellationToken);
|
||||
await FindByPairAsync(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId, cancellationToken);
|
||||
|
||||
if (existing == null)
|
||||
{
|
||||
var created = new PeerOplogConfirmation
|
||||
{
|
||||
DatasetId = normalizedDatasetId,
|
||||
PeerNodeId = peerNodeId,
|
||||
SourceNodeId = RegistrationSourceNodeId,
|
||||
ConfirmedWall = 0,
|
||||
@@ -58,7 +85,9 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
IsActive = true
|
||||
};
|
||||
|
||||
await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId),
|
||||
await UpsertAsync(
|
||||
created,
|
||||
SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId),
|
||||
cancellationToken);
|
||||
|
||||
_logger.LogDebug("Registered peer confirmation tracking for {PeerNodeId} ({Address}, {Type}).", peerNodeId,
|
||||
@@ -71,7 +100,8 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
existing.IsActive = true;
|
||||
existing.LastConfirmedUtcMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
RecordId recordId =
|
||||
existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId);
|
||||
existing.Id ??
|
||||
SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, RegistrationSourceNodeId, normalizedDatasetId);
|
||||
await UpsertAsync(existing, recordId, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -83,19 +113,33 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
string hash,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await UpdateConfirmationAsync(peerNodeId, sourceNodeId, timestamp, hash, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateConfirmationAsync(
|
||||
string peerNodeId,
|
||||
string sourceNodeId,
|
||||
HlcTimestamp timestamp,
|
||||
string hash,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
if (string.IsNullOrWhiteSpace(peerNodeId))
|
||||
throw new ArgumentException("Peer node id is required.", nameof(peerNodeId));
|
||||
|
||||
if (string.IsNullOrWhiteSpace(sourceNodeId))
|
||||
throw new ArgumentException("Source node id is required.", nameof(sourceNodeId));
|
||||
|
||||
var existing = await FindByPairAsync(peerNodeId, sourceNodeId, cancellationToken);
|
||||
var existing = await FindByPairAsync(peerNodeId, sourceNodeId, normalizedDatasetId, cancellationToken);
|
||||
long nowMs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||
|
||||
if (existing == null)
|
||||
{
|
||||
var created = new PeerOplogConfirmation
|
||||
{
|
||||
DatasetId = normalizedDatasetId,
|
||||
PeerNodeId = peerNodeId,
|
||||
SourceNodeId = sourceNodeId,
|
||||
ConfirmedWall = timestamp.PhysicalTime,
|
||||
@@ -104,7 +148,7 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
LastConfirmedUtc = DateTimeOffset.FromUnixTimeMilliseconds(nowMs),
|
||||
IsActive = true
|
||||
};
|
||||
await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId),
|
||||
await UpsertAsync(created, SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId),
|
||||
cancellationToken);
|
||||
return;
|
||||
}
|
||||
@@ -122,7 +166,7 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
existing.LastConfirmedUtcMs = nowMs;
|
||||
existing.IsActive = true;
|
||||
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId);
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId);
|
||||
await UpsertAsync(existing, recordId, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -130,7 +174,15 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
public override async Task<IEnumerable<PeerOplogConfirmation>> GetConfirmationsAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
return await GetConfirmationsAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<PeerOplogConfirmation>> GetConfirmationsAsync(
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all
|
||||
.Where(c => !string.Equals(c.SourceNodeId, RegistrationSourceNodeId, StringComparison.Ordinal))
|
||||
.Select(c => c.ToDomain())
|
||||
@@ -141,11 +193,20 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
public override async Task<IEnumerable<PeerOplogConfirmation>> GetConfirmationsForPeerAsync(
|
||||
string peerNodeId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await GetConfirmationsForPeerAsync(peerNodeId, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<PeerOplogConfirmation>> GetConfirmationsForPeerAsync(
|
||||
string peerNodeId,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(peerNodeId))
|
||||
throw new ArgumentException("Peer node id is required.", nameof(peerNodeId));
|
||||
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all
|
||||
.Where(c => string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal) &&
|
||||
!string.Equals(c.SourceNodeId, RegistrationSourceNodeId, StringComparison.Ordinal))
|
||||
@@ -156,10 +217,18 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
/// <inheritdoc />
|
||||
public override async Task RemovePeerTrackingAsync(string peerNodeId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await RemovePeerTrackingAsync(peerNodeId, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task RemovePeerTrackingAsync(string peerNodeId, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
if (string.IsNullOrWhiteSpace(peerNodeId))
|
||||
throw new ArgumentException("Peer node id is required.", nameof(peerNodeId));
|
||||
|
||||
var matches = (await SelectAllAsync(cancellationToken))
|
||||
var matches = (await SelectAllAsync(normalizedDatasetId, cancellationToken))
|
||||
.Where(c => string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal))
|
||||
.ToList();
|
||||
|
||||
@@ -173,7 +242,11 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
match.IsActive = false;
|
||||
match.LastConfirmedUtcMs = nowMs;
|
||||
|
||||
RecordId recordId = match.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(match.PeerNodeId, match.SourceNodeId);
|
||||
RecordId recordId = match.Id ??
|
||||
SurrealStoreRecordIds.PeerOplogConfirmation(
|
||||
match.PeerNodeId,
|
||||
match.SourceNodeId,
|
||||
normalizedDatasetId);
|
||||
await UpsertAsync(match, recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -182,7 +255,15 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
public override async Task<IEnumerable<string>> GetActiveTrackedPeersAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
return await GetActiveTrackedPeersAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<string>> GetActiveTrackedPeersAsync(
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all
|
||||
.Where(c => c.IsActive)
|
||||
.Select(c => c.PeerNodeId)
|
||||
@@ -193,14 +274,45 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
/// <inheritdoc />
|
||||
public override async Task DropAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient.Delete(CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable, cancellationToken);
|
||||
await DropAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops all peer confirmation rows for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
RecordId recordId = row.Id ??
|
||||
SurrealStoreRecordIds.PeerOplogConfirmation(
|
||||
row.PeerNodeId,
|
||||
row.SourceNodeId,
|
||||
NormalizeDatasetId(datasetId));
|
||||
await _surrealClient.Delete(recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<PeerOplogConfirmation>> ExportAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
return all.Select(c => c.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports peer confirmations for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>The exported confirmations.</returns>
|
||||
public async Task<IEnumerable<PeerOplogConfirmation>> ExportAsync(string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all.Select(c => c.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
@@ -208,11 +320,25 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
public override async Task ImportAsync(IEnumerable<PeerOplogConfirmation> items,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await ImportAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports peer confirmation rows into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The confirmation items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task ImportAsync(IEnumerable<PeerOplogConfirmation> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, cancellationToken);
|
||||
item.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId, cancellationToken);
|
||||
RecordId recordId =
|
||||
existing?.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId);
|
||||
existing?.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId);
|
||||
await UpsertAsync(item, recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -221,12 +347,26 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
public override async Task MergeAsync(IEnumerable<PeerOplogConfirmation> items,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await MergeAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges peer confirmations into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">The confirmation items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task MergeAsync(IEnumerable<PeerOplogConfirmation> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, cancellationToken);
|
||||
item.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByPairAsync(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId, cancellationToken);
|
||||
if (existing == null)
|
||||
{
|
||||
await UpsertAsync(item, SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId),
|
||||
await UpsertAsync(item, SurrealStoreRecordIds.PeerOplogConfirmation(item.PeerNodeId, item.SourceNodeId, normalizedDatasetId),
|
||||
cancellationToken);
|
||||
continue;
|
||||
}
|
||||
@@ -259,7 +399,11 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
if (!changed) continue;
|
||||
|
||||
RecordId recordId =
|
||||
existing.Id ?? SurrealStoreRecordIds.PeerOplogConfirmation(existing.PeerNodeId, existing.SourceNodeId);
|
||||
existing.Id ??
|
||||
SurrealStoreRecordIds.PeerOplogConfirmation(
|
||||
existing.PeerNodeId,
|
||||
existing.SourceNodeId,
|
||||
normalizedDatasetId);
|
||||
await UpsertAsync(existing, recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -288,27 +432,37 @@ public class SurrealPeerOplogConfirmationStore : PeerOplogConfirmationStore
|
||||
await _schemaInitializer.EnsureInitializedAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<List<SurrealPeerOplogConfirmationRecord>> SelectAllAsync(CancellationToken cancellationToken)
|
||||
private async Task<List<SurrealPeerOplogConfirmationRecord>> SelectAllAsync(string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
var rows = await _surrealClient.Select<SurrealPeerOplogConfirmationRecord>(
|
||||
CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable,
|
||||
cancellationToken);
|
||||
return rows?.ToList() ?? [];
|
||||
return rows?
|
||||
.Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId))
|
||||
.ToList()
|
||||
?? [];
|
||||
}
|
||||
|
||||
private async Task<SurrealPeerOplogConfirmationRecord?> FindByPairAsync(string peerNodeId, string sourceNodeId,
|
||||
private async Task<SurrealPeerOplogConfirmationRecord?> FindByPairAsync(
|
||||
string peerNodeId,
|
||||
string sourceNodeId,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.PeerOplogConfirmation(peerNodeId, sourceNodeId, normalizedDatasetId);
|
||||
var deterministic = await _surrealClient.Select<SurrealPeerOplogConfirmationRecord>(deterministicId, cancellationToken);
|
||||
if (deterministic != null &&
|
||||
MatchesDataset(deterministic.DatasetId, normalizedDatasetId) &&
|
||||
string.Equals(deterministic.PeerNodeId, peerNodeId, StringComparison.Ordinal) &&
|
||||
string.Equals(deterministic.SourceNodeId, sourceNodeId, StringComparison.Ordinal))
|
||||
return deterministic;
|
||||
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
return all.FirstOrDefault(c =>
|
||||
string.Equals(c.PeerNodeId, peerNodeId, StringComparison.Ordinal) &&
|
||||
string.Equals(c.SourceNodeId, sourceNodeId, StringComparison.Ordinal));
|
||||
|
||||
@@ -8,6 +8,7 @@ namespace ZB.MOM.WW.CBDDC.Persistence.Surreal;
|
||||
|
||||
public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
{
|
||||
private const string PrimaryDatasetId = DatasetId.Primary;
|
||||
private readonly ILogger<SurrealSnapshotMetadataStore> _logger;
|
||||
private readonly ICBDDCSurrealSchemaInitializer _schemaInitializer;
|
||||
private readonly ISurrealDbClient _surrealClient;
|
||||
@@ -29,17 +30,56 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
_logger = logger ?? NullLogger<SurrealSnapshotMetadataStore>.Instance;
|
||||
}
|
||||
|
||||
private static string NormalizeDatasetId(string? datasetId)
|
||||
{
|
||||
return DatasetId.Normalize(datasetId);
|
||||
}
|
||||
|
||||
private static bool MatchesDataset(string? recordDatasetId, string datasetId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(recordDatasetId))
|
||||
return string.Equals(datasetId, PrimaryDatasetId, StringComparison.Ordinal);
|
||||
|
||||
return string.Equals(NormalizeDatasetId(recordDatasetId), datasetId, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task DropAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
await _surrealClient.Delete(CBDDCSurrealSchemaNames.SnapshotMetadataTable, cancellationToken);
|
||||
await DropAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops snapshot metadata rows for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task DropAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
RecordId recordId = row.Id ?? SurrealStoreRecordIds.SnapshotMetadata(row.NodeId, NormalizeDatasetId(datasetId));
|
||||
await _surrealClient.Delete(recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<IEnumerable<SnapshotMetadata>> ExportAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(PrimaryDatasetId, cancellationToken);
|
||||
return all.Select(m => m.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exports snapshot metadata rows for a dataset.
|
||||
/// </summary>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
/// <returns>Dataset-scoped snapshot metadata rows.</returns>
|
||||
public async Task<IEnumerable<SnapshotMetadata>> ExportAsync(string datasetId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var all = await SelectAllAsync(NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return all.Select(m => m.ToDomain()).ToList();
|
||||
}
|
||||
|
||||
@@ -47,14 +87,32 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
public override async Task<SnapshotMetadata?> GetSnapshotMetadataAsync(string nodeId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(nodeId, cancellationToken);
|
||||
var existing = await FindByNodeIdAsync(nodeId, PrimaryDatasetId, cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<SnapshotMetadata?> GetSnapshotMetadataAsync(
|
||||
string nodeId,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(nodeId, NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return existing?.ToDomain();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task<string?> GetSnapshotHashAsync(string nodeId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(nodeId, cancellationToken);
|
||||
var existing = await FindByNodeIdAsync(nodeId, PrimaryDatasetId, cancellationToken);
|
||||
return existing?.Hash;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<string?> GetSnapshotHashAsync(string nodeId, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(nodeId, NormalizeDatasetId(datasetId), cancellationToken);
|
||||
return existing?.Hash;
|
||||
}
|
||||
|
||||
@@ -62,10 +120,24 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
public override async Task ImportAsync(IEnumerable<SnapshotMetadata> items,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await ImportAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports snapshot metadata rows into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">Snapshot metadata items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task ImportAsync(IEnumerable<SnapshotMetadata> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
foreach (var item in items)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(item.NodeId, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(item.NodeId);
|
||||
item.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByNodeIdAsync(item.NodeId, normalizedDatasetId, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(item.NodeId, normalizedDatasetId);
|
||||
await UpsertAsync(item, recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -74,20 +146,45 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
public override async Task InsertSnapshotMetadataAsync(SnapshotMetadata metadata,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(metadata.NodeId, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId);
|
||||
await InsertSnapshotMetadataAsync(metadata, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task InsertSnapshotMetadataAsync(
|
||||
SnapshotMetadata metadata,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
metadata.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByNodeIdAsync(metadata.NodeId, normalizedDatasetId, cancellationToken);
|
||||
RecordId recordId = existing?.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId);
|
||||
await UpsertAsync(metadata, recordId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task MergeAsync(IEnumerable<SnapshotMetadata> items, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await MergeAsync(items, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges snapshot metadata rows into a dataset.
|
||||
/// </summary>
|
||||
/// <param name="items">Snapshot metadata items.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <param name="cancellationToken">A cancellation token.</param>
|
||||
public async Task MergeAsync(IEnumerable<SnapshotMetadata> items, string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
foreach (var metadata in items)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(metadata.NodeId, cancellationToken);
|
||||
metadata.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByNodeIdAsync(metadata.NodeId, normalizedDatasetId, cancellationToken);
|
||||
if (existing == null)
|
||||
{
|
||||
await UpsertAsync(metadata, SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId), cancellationToken);
|
||||
await UpsertAsync(metadata, SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId), cancellationToken);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -96,7 +193,7 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
metadata.TimestampLogicalCounter <= existing.TimestampLogicalCounter))
|
||||
continue;
|
||||
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId);
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(metadata.NodeId, normalizedDatasetId);
|
||||
await UpsertAsync(metadata, recordId, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -105,10 +202,21 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
public override async Task UpdateSnapshotMetadataAsync(SnapshotMetadata existingMeta,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var existing = await FindByNodeIdAsync(existingMeta.NodeId, cancellationToken);
|
||||
await UpdateSnapshotMetadataAsync(existingMeta, PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpdateSnapshotMetadataAsync(
|
||||
SnapshotMetadata existingMeta,
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
existingMeta.DatasetId = normalizedDatasetId;
|
||||
var existing = await FindByNodeIdAsync(existingMeta.NodeId, normalizedDatasetId, cancellationToken);
|
||||
if (existing == null) return;
|
||||
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(existingMeta.NodeId);
|
||||
RecordId recordId = existing.Id ?? SurrealStoreRecordIds.SnapshotMetadata(existingMeta.NodeId, normalizedDatasetId);
|
||||
await UpsertAsync(existingMeta, recordId, cancellationToken);
|
||||
}
|
||||
|
||||
@@ -116,7 +224,15 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
public override async Task<IEnumerable<SnapshotMetadata>> GetAllSnapshotMetadataAsync(
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await ExportAsync(cancellationToken);
|
||||
return await ExportAsync(PrimaryDatasetId, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IEnumerable<SnapshotMetadata>> GetAllSnapshotMetadataAsync(
|
||||
string datasetId,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
return await ExportAsync(datasetId, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task UpsertAsync(SnapshotMetadata metadata, RecordId recordId, CancellationToken cancellationToken)
|
||||
@@ -133,25 +249,33 @@ public class SurrealSnapshotMetadataStore : SnapshotMetadataStore
|
||||
await _schemaInitializer.EnsureInitializedAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private async Task<List<SurrealSnapshotMetadataRecord>> SelectAllAsync(CancellationToken cancellationToken)
|
||||
private async Task<List<SurrealSnapshotMetadataRecord>> SelectAllAsync(string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
var rows = await _surrealClient.Select<SurrealSnapshotMetadataRecord>(
|
||||
CBDDCSurrealSchemaNames.SnapshotMetadataTable,
|
||||
cancellationToken);
|
||||
return rows?.ToList() ?? [];
|
||||
return rows?
|
||||
.Where(row => MatchesDataset(row.DatasetId, normalizedDatasetId))
|
||||
.ToList()
|
||||
?? [];
|
||||
}
|
||||
|
||||
private async Task<SurrealSnapshotMetadataRecord?> FindByNodeIdAsync(string nodeId, CancellationToken cancellationToken)
|
||||
private async Task<SurrealSnapshotMetadataRecord?> FindByNodeIdAsync(string nodeId, string datasetId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string normalizedDatasetId = NormalizeDatasetId(datasetId);
|
||||
await EnsureReadyAsync(cancellationToken);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.SnapshotMetadata(nodeId);
|
||||
RecordId deterministicId = SurrealStoreRecordIds.SnapshotMetadata(nodeId, normalizedDatasetId);
|
||||
var deterministic = await _surrealClient.Select<SurrealSnapshotMetadataRecord>(deterministicId, cancellationToken);
|
||||
if (deterministic != null &&
|
||||
MatchesDataset(deterministic.DatasetId, normalizedDatasetId) &&
|
||||
string.Equals(deterministic.NodeId, nodeId, StringComparison.Ordinal))
|
||||
return deterministic;
|
||||
|
||||
var all = await SelectAllAsync(cancellationToken);
|
||||
var all = await SelectAllAsync(normalizedDatasetId, cancellationToken);
|
||||
return all.FirstOrDefault(m => string.Equals(m.NodeId, nodeId, StringComparison.Ordinal));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,22 +26,28 @@ internal static class SurrealStoreRecordIds
|
||||
/// </summary>
|
||||
/// <param name="collection">The document collection name.</param>
|
||||
/// <param name="key">The document key.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <returns>The SurrealDB record identifier.</returns>
|
||||
public static RecordId DocumentMetadata(string collection, string key)
|
||||
public static RecordId DocumentMetadata(string collection, string key, string datasetId)
|
||||
{
|
||||
string normalizedDatasetId = DatasetId.Normalize(datasetId);
|
||||
return RecordId.From(
|
||||
CBDDCSurrealSchemaNames.DocumentMetadataTable,
|
||||
CompositeKey("docmeta", collection, key));
|
||||
CompositeKey("docmeta", normalizedDatasetId, collection, key));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the record identifier for snapshot metadata.
|
||||
/// </summary>
|
||||
/// <param name="nodeId">The node identifier.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <returns>The SurrealDB record identifier.</returns>
|
||||
public static RecordId SnapshotMetadata(string nodeId)
|
||||
public static RecordId SnapshotMetadata(string nodeId, string datasetId)
|
||||
{
|
||||
return RecordId.From(CBDDCSurrealSchemaNames.SnapshotMetadataTable, nodeId);
|
||||
string normalizedDatasetId = DatasetId.Normalize(datasetId);
|
||||
return RecordId.From(
|
||||
CBDDCSurrealSchemaNames.SnapshotMetadataTable,
|
||||
CompositeKey("snapshotmeta", normalizedDatasetId, nodeId));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -59,12 +65,14 @@ internal static class SurrealStoreRecordIds
|
||||
/// </summary>
|
||||
/// <param name="peerNodeId">The peer node identifier.</param>
|
||||
/// <param name="sourceNodeId">The source node identifier.</param>
|
||||
/// <param name="datasetId">The dataset identifier.</param>
|
||||
/// <returns>The SurrealDB record identifier.</returns>
|
||||
public static RecordId PeerOplogConfirmation(string peerNodeId, string sourceNodeId)
|
||||
public static RecordId PeerOplogConfirmation(string peerNodeId, string sourceNodeId, string datasetId)
|
||||
{
|
||||
string normalizedDatasetId = DatasetId.Normalize(datasetId);
|
||||
return RecordId.From(
|
||||
CBDDCSurrealSchemaNames.PeerOplogConfirmationsTable,
|
||||
CompositeKey("peerconfirm", peerNodeId, sourceNodeId));
|
||||
CompositeKey("peerconfirm", normalizedDatasetId, peerNodeId, sourceNodeId));
|
||||
}
|
||||
|
||||
private static string CompositeKey(string prefix, string first, string second)
|
||||
@@ -72,10 +80,22 @@ internal static class SurrealStoreRecordIds
|
||||
byte[] bytes = Encoding.UTF8.GetBytes($"{prefix}\n{first}\n{second}");
|
||||
return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static string CompositeKey(string prefix, string first, string second, string third)
|
||||
{
|
||||
byte[] bytes = Encoding.UTF8.GetBytes($"{prefix}\n{first}\n{second}\n{third}");
|
||||
return Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class SurrealOplogRecord : Record
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("datasetId")]
|
||||
public string DatasetId { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the collection name.
|
||||
/// </summary>
|
||||
@@ -133,6 +153,12 @@ internal sealed class SurrealOplogRecord : Record
|
||||
|
||||
internal sealed class SurrealDocumentMetadataRecord : Record
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("datasetId")]
|
||||
public string DatasetId { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the collection name.
|
||||
/// </summary>
|
||||
@@ -205,6 +231,12 @@ internal sealed class SurrealRemotePeerRecord : Record
|
||||
|
||||
internal sealed class SurrealPeerOplogConfirmationRecord : Record
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("datasetId")]
|
||||
public string DatasetId { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the peer node identifier.
|
||||
/// </summary>
|
||||
@@ -250,6 +282,12 @@ internal sealed class SurrealPeerOplogConfirmationRecord : Record
|
||||
|
||||
internal sealed class SurrealSnapshotMetadataRecord : Record
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the dataset identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("datasetId")]
|
||||
public string DatasetId { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the node identifier.
|
||||
/// </summary>
|
||||
@@ -286,6 +324,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new SurrealOplogRecord
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(entry.DatasetId),
|
||||
Collection = entry.Collection,
|
||||
Key = entry.Key,
|
||||
Operation = (int)entry.Operation,
|
||||
@@ -316,7 +355,8 @@ internal static class SurrealStoreRecordMappers
|
||||
payload,
|
||||
new HlcTimestamp(record.TimestampPhysicalTime, record.TimestampLogicalCounter, record.TimestampNodeId),
|
||||
record.PreviousHash,
|
||||
record.Hash);
|
||||
record.Hash,
|
||||
record.DatasetId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -328,6 +368,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new SurrealDocumentMetadataRecord
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(metadata.DatasetId),
|
||||
Collection = metadata.Collection,
|
||||
Key = metadata.Key,
|
||||
HlcPhysicalTime = metadata.UpdatedAt.PhysicalTime,
|
||||
@@ -348,7 +389,8 @@ internal static class SurrealStoreRecordMappers
|
||||
record.Collection,
|
||||
record.Key,
|
||||
new HlcTimestamp(record.HlcPhysicalTime, record.HlcLogicalCounter, record.HlcNodeId),
|
||||
record.IsDeleted);
|
||||
record.IsDeleted,
|
||||
record.DatasetId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -401,6 +443,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new SurrealPeerOplogConfirmationRecord
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(confirmation.DatasetId),
|
||||
PeerNodeId = confirmation.PeerNodeId,
|
||||
SourceNodeId = confirmation.SourceNodeId,
|
||||
ConfirmedWall = confirmation.ConfirmedWall,
|
||||
@@ -420,6 +463,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new PeerOplogConfirmation
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(record.DatasetId),
|
||||
PeerNodeId = record.PeerNodeId,
|
||||
SourceNodeId = record.SourceNodeId,
|
||||
ConfirmedWall = record.ConfirmedWall,
|
||||
@@ -439,6 +483,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new SurrealSnapshotMetadataRecord
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(metadata.DatasetId),
|
||||
NodeId = metadata.NodeId,
|
||||
TimestampPhysicalTime = metadata.TimestampPhysicalTime,
|
||||
TimestampLogicalCounter = metadata.TimestampLogicalCounter,
|
||||
@@ -455,6 +500,7 @@ internal static class SurrealStoreRecordMappers
|
||||
{
|
||||
return new SnapshotMetadata
|
||||
{
|
||||
DatasetId = DatasetId.Normalize(record.DatasetId),
|
||||
NodeId = record.NodeId,
|
||||
TimestampPhysicalTime = record.TimestampPhysicalTime,
|
||||
TimestampLogicalCounter = record.TimestampLogicalCounter,
|
||||
|
||||
Reference in New Issue
Block a user