using ZB.MOM.WW.CBDD.Bson; using ZB.MOM.WW.CBDD.Core.Collections; using ZB.MOM.WW.CBDD.Core.Storage; using ZB.MOM.WW.CBDD.Core.Transactions; using ZB.MOM.WW.CBDD.Core.Indexing.Internal; using System; using System.Linq; using System.Collections.Generic; namespace ZB.MOM.WW.CBDD.Core.Indexing; /// /// Represents a secondary (non-primary) index on a document collection. /// Provides a high-level, strongly-typed wrapper around the low-level BTreeIndex. /// Handles automatic key extraction from documents using compiled expressions. /// /// Primary key type /// Document type public sealed class CollectionSecondaryIndex : IDisposable where T : class { private readonly CollectionIndexDefinition _definition; private readonly BTreeIndex? _btreeIndex; private readonly VectorSearchIndex? _vectorIndex; private readonly RTreeIndex? _spatialIndex; private readonly IDocumentMapper _mapper; private bool _disposed; /// /// Gets the index definition /// public CollectionIndexDefinition Definition => _definition; /// /// Gets the underlying BTree index (for advanced scenarios) /// public BTreeIndex? BTreeIndex => _btreeIndex; /// /// Gets the root page identifier for the underlying index structure. /// public uint RootPageId => _btreeIndex?.RootPageId ?? _vectorIndex?.RootPageId ?? _spatialIndex?.RootPageId ?? 0; /// /// Initializes a new instance of the class. /// /// The index definition. /// The storage engine. /// The document mapper. /// The existing root page ID, or 0 to create a new one. public CollectionSecondaryIndex( CollectionIndexDefinition definition, StorageEngine storage, IDocumentMapper mapper, uint rootPageId = 0) : this(definition, (IStorageEngine)storage, mapper, rootPageId) { } /// /// Initializes a new instance of the class from index storage abstractions. /// /// The index definition. /// The index storage abstraction. /// The document mapper. /// The existing root page identifier, if any. internal CollectionSecondaryIndex( CollectionIndexDefinition definition, IIndexStorage storage, IDocumentMapper mapper, uint rootPageId = 0) { _definition = definition ?? throw new ArgumentNullException(nameof(definition)); _mapper = mapper ?? throw new ArgumentNullException(nameof(mapper)); var indexOptions = definition.ToIndexOptions(); if (indexOptions.Type == IndexType.Vector) { _vectorIndex = new VectorSearchIndex(storage, indexOptions, rootPageId); _btreeIndex = null; _spatialIndex = null; } else if (indexOptions.Type == IndexType.Spatial) { _spatialIndex = new RTreeIndex(storage, indexOptions, rootPageId); _btreeIndex = null; _vectorIndex = null; } else { _btreeIndex = new BTreeIndex(storage, indexOptions, rootPageId); _vectorIndex = null; _spatialIndex = null; } } /// /// Inserts a document into this index /// /// Document to index /// Physical location of the document /// Optional transaction public void Insert(T document, DocumentLocation location, ITransaction transaction) { if (document == null) throw new ArgumentNullException(nameof(document)); // Extract key using compiled selector (fast!) var keyValue = _definition.KeySelector(document); if (keyValue == null) return; // Skip null keys if (_vectorIndex != null) { // Vector Index Support if (keyValue is float[] singleVector) { _vectorIndex.Insert(singleVector, location, transaction); } else if (keyValue is IEnumerable vectors) { foreach (var v in vectors) { _vectorIndex.Insert(v, location, transaction); } } } else if (_spatialIndex != null) { // Geospatial Index Support if (keyValue is ValueTuple t) { _spatialIndex.Insert(GeoBox.FromPoint(new GeoPoint(t.Item1, t.Item2)), location, transaction); } } else if (_btreeIndex != null) { // BTree Index logic var userKey = ConvertToIndexKey(keyValue); var documentId = _mapper.GetId(document); var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId)); _btreeIndex.Insert(compositeKey, location, transaction?.TransactionId); } } /// /// Updates a document in this index (delete old, insert new). /// Only updates if the indexed key has changed. /// /// Old version of document /// New version of document /// Physical location of old document /// Physical location of new document /// Optional transaction public void Update(T oldDocument, T newDocument, DocumentLocation oldLocation, DocumentLocation newLocation, ITransaction transaction) { if (oldDocument == null) throw new ArgumentNullException(nameof(oldDocument)); if (newDocument == null) throw new ArgumentNullException(nameof(newDocument)); // Extract keys from both versions var oldKey = _definition.KeySelector(oldDocument); var newKey = _definition.KeySelector(newDocument); // If keys are the same, no index update needed (optimization) if (Equals(oldKey, newKey)) return; var documentId = _mapper.GetId(oldDocument); // Delete old entry if it had a key if (oldKey != null) { var oldUserKey = ConvertToIndexKey(oldKey); var oldCompositeKey = CreateCompositeKey(oldUserKey, _mapper.ToIndexKey(documentId)); _btreeIndex?.Delete(oldCompositeKey, oldLocation, transaction?.TransactionId); } // Insert new entry if it has a key if (newKey != null) { var newUserKey = ConvertToIndexKey(newKey); var newCompositeKey = CreateCompositeKey(newUserKey, _mapper.ToIndexKey(documentId)); _btreeIndex?.Insert(newCompositeKey, newLocation, transaction?.TransactionId); } } /// /// Deletes a document from this index /// /// Document to remove from index /// Physical location of the document /// Optional transaction public void Delete(T document, DocumentLocation location, ITransaction transaction) { if (document == null) throw new ArgumentNullException(nameof(document)); // Extract key var keyValue = _definition.KeySelector(document); if (keyValue == null) return; // Nothing to delete var userKey = ConvertToIndexKey(keyValue); var documentId = _mapper.GetId(document); // Create composite key and delete var compositeKey = CreateCompositeKey(userKey, _mapper.ToIndexKey(documentId)); _btreeIndex?.Delete(compositeKey, location, transaction?.TransactionId); } /// /// Seeks a single document by exact key match (O(log n)) /// /// Key value to seek /// Optional transaction to read uncommitted changes /// Document location if found, null otherwise public DocumentLocation? Seek(object key, ITransaction? transaction = null) { if (key == null) return null; if (_vectorIndex != null && key is float[] query) { return _vectorIndex.Search(query, 1, transaction: transaction).FirstOrDefault().Location; } if (_btreeIndex != null) { var userKey = ConvertToIndexKey(key); var minComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: true); var maxComposite = CreateCompositeKeyBoundary(userKey, useMinObjectId: false); var firstEntry = _btreeIndex.Range(minComposite, maxComposite, IndexDirection.Forward, transaction?.TransactionId).FirstOrDefault(); return firstEntry.Location.PageId == 0 ? null : (DocumentLocation?)firstEntry.Location; } return null; } /// /// Performs a vector nearest-neighbor search. /// /// The query vector. /// The number of results to return. /// The search breadth parameter. /// Optional transaction. /// The matching vector search results. public IEnumerable VectorSearch(float[] query, int k, int efSearch = 100, ITransaction? transaction = null) { if (_vectorIndex == null) throw new InvalidOperationException("This index is not a vector index."); return _vectorIndex.Search(query, k, efSearch, transaction); } /// /// Performs geospatial distance search /// /// The center point. /// The search radius in kilometers. /// Optional transaction. public IEnumerable Near((double Latitude, double Longitude) center, double radiusKm, ITransaction? transaction = null) { if (_spatialIndex == null) throw new InvalidOperationException("This index is not a spatial index."); var queryBox = SpatialMath.BoundingBox(center.Latitude, center.Longitude, radiusKm); foreach (var loc in _spatialIndex.Search(queryBox, transaction)) { yield return loc; } } /// /// Performs geospatial bounding box search /// /// The minimum latitude/longitude corner. /// The maximum latitude/longitude corner. /// Optional transaction. public IEnumerable Within((double Latitude, double Longitude) min, (double Latitude, double Longitude) max, ITransaction? transaction = null) { if (_spatialIndex == null) throw new InvalidOperationException("This index is not a spatial index."); var area = new GeoBox(min.Latitude, min.Longitude, max.Latitude, max.Longitude); return _spatialIndex.Search(area, transaction); } /// /// Scans a range of keys (O(log n + k) where k is result count) /// /// Minimum key (inclusive), null for unbounded /// Maximum key (inclusive), null for unbounded /// Scan direction. /// Optional transaction to read uncommitted changes /// Enumerable of document locations in key order public IEnumerable Range(object? minKey, object? maxKey, IndexDirection direction = IndexDirection.Forward, ITransaction? transaction = null) { if (_btreeIndex == null) yield break; // Handle unbounded ranges IndexKey actualMinKey; IndexKey actualMaxKey; if (minKey == null && maxKey == null) { // Full scan - use extreme values actualMinKey = new IndexKey(new byte[0]); // Empty = smallest actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray()); // Max bytes } else if (minKey == null) { actualMinKey = new IndexKey(new byte[0]); var userMaxKey = ConvertToIndexKey(maxKey!); actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false); // Max boundary } else if (maxKey == null) { var userMinKey = ConvertToIndexKey(minKey); actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true); // Min boundary actualMaxKey = new IndexKey(Enumerable.Repeat((byte)0xFF, 255).ToArray()); } else { // Both bounds specified var userMinKey = ConvertToIndexKey(minKey); var userMaxKey = ConvertToIndexKey(maxKey); // Create composite boundaries: // Min: (userMinKey, ObjectId.Empty) - captures all docs with key >= userMinKey // Max: (userMaxKey, ObjectId.MaxValue) - captures all docs with key <= userMaxKey actualMinKey = CreateCompositeKeyBoundary(userMinKey, useMinObjectId: true); actualMaxKey = CreateCompositeKeyBoundary(userMaxKey, useMinObjectId: false); } // Use BTreeIndex.Range with WAL-aware reads and direction // Extract DocumentLocation from each entry foreach (var entry in _btreeIndex.Range(actualMinKey, actualMaxKey, direction, transaction?.TransactionId)) { yield return entry.Location; } } /// /// Gets statistics about this index /// public CollectionIndexInfo GetInfo() { return new CollectionIndexInfo { Name = _definition.Name, PropertyPaths = _definition.PropertyPaths, IsUnique = _definition.IsUnique, Type = _definition.Type, IsPrimary = _definition.IsPrimary, EstimatedDocumentCount = 0, // TODO: Track or calculate document count EstimatedSizeBytes = 0 // TODO: Calculate index size }; } #region Composite Key Support (SQLite-style for Duplicate Keys) /// /// Creates a composite key by concatenating user key with document ID. /// This allows duplicate user keys while maintaining BTree uniqueness. /// Format: [UserKeyBytes] + [DocumentIdKey] /// private IndexKey CreateCompositeKey(IndexKey userKey, IndexKey documentIdKey) { // Allocate buffer: user key + document ID key length var compositeBytes = new byte[userKey.Data.Length + documentIdKey.Data.Length]; // Copy user key userKey.Data.CopyTo(compositeBytes.AsSpan(0, userKey.Data.Length)); // Append document ID key documentIdKey.Data.CopyTo(compositeBytes.AsSpan(userKey.Data.Length)); return new IndexKey(compositeBytes); } /// /// Creates a composite key for range query boundary. /// Uses MIN or MAX ID representation to capture all documents with the user key. /// private IndexKey CreateCompositeKeyBoundary(IndexKey userKey, bool useMinObjectId) { // For range boundaries, we use an empty key for Min and a very large key for Max // to wrap around all possible IDs for this user key. IndexKey idBoundary = useMinObjectId ? new IndexKey(Array.Empty()) : new IndexKey(Enumerable.Repeat((byte)0xFF, 16).ToArray()); // Using 16 as a safe max for GUID/ObjectId return CreateCompositeKey(userKey, idBoundary); } /// /// Extracts the original user key from a composite key by removing the ObjectId suffix. /// Used when we need to return the original indexed value. /// private IndexKey ExtractUserKey(IndexKey compositeKey) { // Composite key = UserKey + ObjectId(12 bytes) var userKeyLength = compositeKey.Data.Length - 12; if (userKeyLength <= 0) return compositeKey; // Fallback for malformed keys var userKeyBytes = compositeKey.Data.Slice(0, userKeyLength); return new IndexKey(userKeyBytes); } #endregion /// /// Converts a CLR value to an IndexKey for BTree storage. /// Supports all common .NET types. /// private IndexKey ConvertToIndexKey(object value) { return value switch { ObjectId objectId => new IndexKey(objectId), string str => new IndexKey(str), int intVal => new IndexKey(intVal), long longVal => new IndexKey(longVal), DateTime dateTime => new IndexKey(dateTime.Ticks), bool boolVal => new IndexKey(boolVal ? 1 : 0), byte[] byteArray => new IndexKey(byteArray), // For compound keys or complex types, use ToString and serialize // TODO: Better compound key serialization _ => new IndexKey(value.ToString() ?? string.Empty) }; } /// /// Releases resources used by this index wrapper. /// public void Dispose() { if (_disposed) return; // BTreeIndex doesn't currently implement IDisposable // Future: may need to flush buffers, close resources _disposed = true; GC.SuppressFinalize(this); } }