From 528939d3a04917769cad4a497c136087444d5c30 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 20 Feb 2026 13:32:10 -0500 Subject: [PATCH] Add Gitea NuGet publish workflow and finalize current storage/index/docs updates. --- .gitea/workflows/nuget-publish.yml | 76 +++++++++++++++++++ README.md | 40 +++++----- .../Collections/DocumentCollection.cs | 27 +++++-- src/CBDD.Core/DocumentDbContext.cs | 60 ++++++++++++--- .../Indexing/CollectionIndexManager.cs | 54 ++----------- .../Storage/StorageEngine.Maintenance.cs | 20 +++-- 6 files changed, 185 insertions(+), 92 deletions(-) create mode 100644 .gitea/workflows/nuget-publish.yml diff --git a/.gitea/workflows/nuget-publish.yml b/.gitea/workflows/nuget-publish.yml new file mode 100644 index 0000000..3d5d567 --- /dev/null +++ b/.gitea/workflows/nuget-publish.yml @@ -0,0 +1,76 @@ +name: NuGet Publish + +on: + push: + branches: + - "**" + +jobs: + build-and-pack: + runs-on: ubuntu-latest + container: + image: mcr.microsoft.com/dotnet/sdk:10.0 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Restore + run: dotnet restore CBDD.slnx + + - name: Build + run: dotnet build CBDD.slnx -c Release --no-restore + + - name: Pack NuGet Packages + run: | + mkdir -p nupkgs + dotnet pack src/CBDD/ZB.MOM.WW.CBDD.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj -c Release -o nupkgs --no-build + + - name: Upload Package Artifacts + uses: actions/upload-artifact@v4 + with: + name: nuget-packages + path: nupkgs/*.nupkg + + publish-to-gitea: + needs: build-and-pack + runs-on: ubuntu-latest + if: ${{ gitea.ref == 'refs/heads/main' && secrets.GITEA_NUGET_USERNAME != '' && secrets.GITEA_NUGET_TOKEN != '' }} + container: + image: mcr.microsoft.com/dotnet/sdk:10.0 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Restore + run: dotnet restore CBDD.slnx + + - name: Build + run: dotnet build CBDD.slnx -c Release --no-restore + + - name: Pack NuGet Packages + run: | + mkdir -p nupkgs + dotnet pack src/CBDD/ZB.MOM.WW.CBDD.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.Bson/ZB.MOM.WW.CBDD.Bson.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.Core/ZB.MOM.WW.CBDD.Core.csproj -c Release -o nupkgs --no-build + dotnet pack src/CBDD.SourceGenerators/ZB.MOM.WW.CBDD.SourceGenerators.csproj -c Release -o nupkgs --no-build + + - name: Configure Gitea NuGet Source + run: | + dotnet nuget add source \ + "https://gitea.dohertylan.com/api/packages/dohertj2/nuget/index.json" \ + --name gitea \ + --username "${{ secrets.GITEA_NUGET_USERNAME }}" \ + --password "${{ secrets.GITEA_NUGET_TOKEN }}" \ + --store-password-in-clear-text + + - name: Publish Packages + run: | + for pkg in nupkgs/*.nupkg; do + if [ -f "$pkg" ] && [[ "$pkg" != *.snupkg ]]; then + dotnet nuget push "$pkg" --source gitea --skip-duplicate + fi + done diff --git a/README.md b/README.md index e54f43e..3f4408a 100755 --- a/README.md +++ b/README.md @@ -10,21 +10,21 @@ CBDD provides a local data layer for services and tools that need transactional - Owning team: CBDD maintainers (repository owner: `@dohertj2`) - Primary support path: open a Gitea issue in this repository with labels `incident` or `bug` -- Escalation path: follow `/Users/dohertj2/Desktop/CBDD/docs/runbook.md` and page the release maintainer listed in the active release PR +- Escalation path: follow [`docs/runbook.md`](docs/runbook.md) and page the release maintainer listed in the active release PR ## Architecture Overview CBDD has four primary layers: -1. Storage and transaction engine (`/Users/dohertj2/Desktop/CBDD/src/CBDD.Core/Storage`, `/Users/dohertj2/Desktop/CBDD/src/CBDD.Core/Transactions`) -2. BSON serialization (`/Users/dohertj2/Desktop/CBDD/src/CBDD.Bson`) -3. Indexing and query execution (`/Users/dohertj2/Desktop/CBDD/src/CBDD.Core/Indexing`, `/Users/dohertj2/Desktop/CBDD/src/CBDD.Core/Query`) -4. Source-generated mapping (`/Users/dohertj2/Desktop/CBDD/src/CBDD.SourceGenerators`) +1. Storage and transaction engine (`src/CBDD.Core/Storage`, `src/CBDD.Core/Transactions`) +2. BSON serialization (`src/CBDD.Bson`) +3. Indexing and query execution (`src/CBDD.Core/Indexing`, `src/CBDD.Core/Query`) +4. Source-generated mapping (`src/CBDD.SourceGenerators`) Detailed architecture material: -- `/Users/dohertj2/Desktop/CBDD/docs/architecture.md` -- `/Users/dohertj2/Desktop/CBDD/RFC.md` -- `/Users/dohertj2/Desktop/CBDD/C-BSON.md` +- [`docs/architecture.md`](docs/architecture.md) +- [`RFC.md`](RFC.md) +- [`C-BSON.md`](C-BSON.md) ## Prerequisites @@ -94,29 +94,29 @@ bash scripts/fitness-check.sh CBDD is released as an internal package. -- Deployment workflow: `/Users/dohertj2/Desktop/CBDD/docs/deployment.md` -- Rollback workflow: `/Users/dohertj2/Desktop/CBDD/docs/deployment.md#rollback-procedure` +- Deployment workflow: [`docs/deployment.md`](docs/deployment.md) +- Rollback workflow: [`docs/deployment.md#rollback-procedure`](docs/deployment.md#rollback-procedure) ## Operations And Incident Response Operational procedures, diagnostics, and escalation are documented in: -- `/Users/dohertj2/Desktop/CBDD/docs/runbook.md` -- `/Users/dohertj2/Desktop/CBDD/docs/troubleshooting.md` +- [`docs/runbook.md`](docs/runbook.md) +- [`docs/troubleshooting.md`](docs/troubleshooting.md) ## Security And Compliance Posture - CBDD relies on host and process-level access controls. -- Sensitive payload classification and handling requirements are defined in `/Users/dohertj2/Desktop/CBDD/docs/security.md`. -- Role and approval requirements are defined in `/Users/dohertj2/Desktop/CBDD/docs/access.md`. +- Sensitive payload classification and handling requirements are defined in [`docs/security.md`](docs/security.md). +- Role and approval requirements are defined in [`docs/access.md`](docs/access.md). ## Troubleshooting Common issues and remediation: -- Build/test environment failures: `/Users/dohertj2/Desktop/CBDD/docs/troubleshooting.md#build-and-test-failures` -- Data-file recovery procedures: `/Users/dohertj2/Desktop/CBDD/docs/troubleshooting.md#data-file-and-recovery-issues` -- Query/index behavior verification: `/Users/dohertj2/Desktop/CBDD/docs/troubleshooting.md#query-and-index-issues` +- Build/test environment failures: [`docs/troubleshooting.md#build-and-test-failures`](docs/troubleshooting.md#build-and-test-failures) +- Data-file recovery procedures: [`docs/troubleshooting.md#data-file-and-recovery-issues`](docs/troubleshooting.md#data-file-and-recovery-issues) +- Query/index behavior verification: [`docs/troubleshooting.md#query-and-index-issues`](docs/troubleshooting.md#query-and-index-issues) ## Change Governance @@ -127,6 +127,6 @@ Common issues and remediation: ## Documentation Index -- Documentation home: `/Users/dohertj2/Desktop/CBDD/docs/README.md` -- Major feature inventory: `/Users/dohertj2/Desktop/CBDD/docs/features/README.md` -- Architecture decisions: `/Users/dohertj2/Desktop/CBDD/docs/adr/0001-storage-engine-and-source-generation.md` +- Documentation home: [`docs/README.md`](docs/README.md) +- Major feature inventory: [`docs/features/README.md`](docs/features/README.md) +- Architecture decisions: [`docs/adr/0001-storage-engine-and-source-generation.md`](docs/adr/0001-storage-engine-and-source-generation.md) diff --git a/src/CBDD.Core/Collections/DocumentCollection.cs b/src/CBDD.Core/Collections/DocumentCollection.cs index e75159d..0b8f0a5 100755 --- a/src/CBDD.Core/Collections/DocumentCollection.cs +++ b/src/CBDD.Core/Collections/DocumentCollection.cs @@ -16,6 +16,7 @@ using System; using System.IO; using System.Diagnostics; using System.Threading; +using ZB.MOM.WW.CBDD.Core; [assembly: InternalsVisibleTo("ZB.MOM.WW.CBDD.Tests")] @@ -48,7 +49,7 @@ public class DocumentCollection : DocumentCollection where T : c /// /// Type of the primary key /// Type of the entity -public partial class DocumentCollection : IDisposable where T : class +public partial class DocumentCollection : IDisposable, ICompactionAwareCollection where T : class { private readonly ITransactionHolder _transactionHolder; private readonly IStorageEngine _storage; @@ -130,15 +131,27 @@ public partial class DocumentCollection : IDisposable where T : class private void RefreshPrimaryIndexRootFromMetadata() { - _indexManager.RefreshFromStorageMetadata(); - - var primaryRootPageId = _indexManager.PrimaryRootPageId; - if (primaryRootPageId == 0) + var metadata = _storage.GetCollectionMetadata(_collectionName); + if (metadata == null || metadata.PrimaryRootPageId == 0) return; - if (primaryRootPageId != _primaryIndex.RootPageId) + if (metadata.PrimaryRootPageId != _primaryIndex.RootPageId) { - _primaryIndex.SetRootPageId(primaryRootPageId); + _primaryIndex.SetRootPageId(metadata.PrimaryRootPageId); + } + } + + void ICompactionAwareCollection.RefreshIndexBindingsAfterCompaction() + { + var metadata = _storage.GetCollectionMetadata(_collectionName); + if (metadata == null) + return; + + _indexManager.RebindFromMetadata(metadata); + + if (metadata.PrimaryRootPageId != 0 && metadata.PrimaryRootPageId != _primaryIndex.RootPageId) + { + _primaryIndex.SetRootPageId(metadata.PrimaryRootPageId); } } diff --git a/src/CBDD.Core/DocumentDbContext.cs b/src/CBDD.Core/DocumentDbContext.cs index 680e6af..7d3b823 100755 --- a/src/CBDD.Core/DocumentDbContext.cs +++ b/src/CBDD.Core/DocumentDbContext.cs @@ -9,10 +9,15 @@ using System.Collections.Generic; using System.Threading.Tasks; using ZB.MOM.WW.CBDD.Bson; -namespace ZB.MOM.WW.CBDD.Core; - -/// -/// Base class for database contexts. +namespace ZB.MOM.WW.CBDD.Core; + +internal interface ICompactionAwareCollection +{ + void RefreshIndexBindingsAfterCompaction(); +} + +/// +/// Base class for database contexts. /// Inherit and add DocumentCollection{T} properties for your entities. /// Use partial class for Source Generator integration. /// @@ -103,6 +108,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde private readonly IReadOnlyDictionary _model; private readonly List _registeredMappers = new(); + private readonly List _compactionAwareCollections = new(); /// /// Gets the concrete storage engine for advanced scenarios in derived contexts. @@ -155,10 +161,14 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde customName = builder?.CollectionName; } - _registeredMappers.Add(mapper); - var collection = new DocumentCollection(_storage, this, mapper, customName); - - // Apply configurations from ModelBuilder + _registeredMappers.Add(mapper); + var collection = new DocumentCollection(_storage, this, mapper, customName); + if (collection is ICompactionAwareCollection compactionAwareCollection) + { + _compactionAwareCollections.Add(compactionAwareCollection); + } + + // Apply configurations from ModelBuilder if (builder != null) { foreach (var indexBuilder in builder.Indexes) @@ -335,7 +345,9 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde if (_disposed) throw new ObjectDisposedException(nameof(DocumentDbContext)); - return Engine.Compact(options); + var stats = Engine.Compact(options); + RefreshCollectionBindingsAfterCompaction(); + return stats; } /// @@ -346,7 +358,7 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde if (_disposed) throw new ObjectDisposedException(nameof(DocumentDbContext)); - return Engine.CompactAsync(options, ct); + return CompactAsyncCore(options, ct); } /// @@ -357,7 +369,9 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde if (_disposed) throw new ObjectDisposedException(nameof(DocumentDbContext)); - return Engine.Vacuum(options); + var stats = Engine.Vacuum(options); + RefreshCollectionBindingsAfterCompaction(); + return stats; } /// @@ -368,7 +382,29 @@ public abstract partial class DocumentDbContext : IDisposable, ITransactionHolde if (_disposed) throw new ObjectDisposedException(nameof(DocumentDbContext)); - return Engine.VacuumAsync(options, ct); + return VacuumAsyncCore(options, ct); + } + + private async Task CompactAsyncCore(CompactionOptions? options, CancellationToken ct) + { + var stats = await Engine.CompactAsync(options, ct); + RefreshCollectionBindingsAfterCompaction(); + return stats; + } + + private async Task VacuumAsyncCore(CompactionOptions? options, CancellationToken ct) + { + var stats = await Engine.VacuumAsync(options, ct); + RefreshCollectionBindingsAfterCompaction(); + return stats; + } + + private void RefreshCollectionBindingsAfterCompaction() + { + foreach (var collection in _compactionAwareCollections) + { + collection.RefreshIndexBindingsAfterCompaction(); + } } /// diff --git a/src/CBDD.Core/Indexing/CollectionIndexManager.cs b/src/CBDD.Core/Indexing/CollectionIndexManager.cs index ebf0cec..53e210d 100755 --- a/src/CBDD.Core/Indexing/CollectionIndexManager.cs +++ b/src/CBDD.Core/Indexing/CollectionIndexManager.cs @@ -505,26 +505,25 @@ public sealed class CollectionIndexManager : IDisposable where T : class public uint PrimaryRootPageId => _metadata.PrimaryRootPageId; /// - /// Refreshes cached metadata and index root bindings from storage. + /// Rebinds cached metadata and index instances from persisted metadata. /// - internal void RefreshFromStorageMetadata() + internal void RebindFromMetadata(CollectionMetadata metadata) { + if (metadata == null) + throw new ArgumentNullException(nameof(metadata)); + lock (_lock) { if (_disposed) throw new ObjectDisposedException(nameof(CollectionIndexManager)); - var latest = _storage.GetCollectionMetadata(_collectionName) ?? new CollectionMetadata { Name = _collectionName }; - if (MetadataEquals(_metadata, latest)) - return; - foreach (var index in _indexes.Values) { try { index.Dispose(); } catch { /* Best effort */ } } _indexes.Clear(); - _metadata = latest; + _metadata = metadata; foreach (var idxMeta in _metadata.Indexes) { @@ -562,47 +561,6 @@ public sealed class CollectionIndexManager : IDisposable where T : class UpdateMetadata(); _storage.SaveCollectionMetadata(_metadata); } - - private static bool MetadataEquals(CollectionMetadata left, CollectionMetadata right) - { - if (!string.Equals(left.Name, right.Name, StringComparison.OrdinalIgnoreCase)) - return false; - - if (left.PrimaryRootPageId != right.PrimaryRootPageId || - left.SchemaRootPageId != right.SchemaRootPageId || - left.Indexes.Count != right.Indexes.Count) - { - return false; - } - - for (var i = 0; i < left.Indexes.Count; i++) - { - var l = left.Indexes[i]; - var r = right.Indexes[i]; - if (!string.Equals(l.Name, r.Name, StringComparison.OrdinalIgnoreCase) || - l.RootPageId != r.RootPageId || - l.Type != r.Type || - l.IsUnique != r.IsUnique || - l.Dimensions != r.Dimensions || - l.Metric != r.Metric) - { - return false; - } - - var lPaths = l.PropertyPaths ?? Array.Empty(); - var rPaths = r.PropertyPaths ?? Array.Empty(); - if (lPaths.Length != rPaths.Length) - return false; - - for (var p = 0; p < lPaths.Length; p++) - { - if (!string.Equals(lPaths[p], rPaths[p], StringComparison.Ordinal)) - return false; - } - } - - return true; - } /// /// Releases resources used by the index manager. diff --git a/src/CBDD.Core/Storage/StorageEngine.Maintenance.cs b/src/CBDD.Core/Storage/StorageEngine.Maintenance.cs index e88a333..1de1c89 100644 --- a/src/CBDD.Core/Storage/StorageEngine.Maintenance.cs +++ b/src/CBDD.Core/Storage/StorageEngine.Maintenance.cs @@ -779,17 +779,28 @@ public sealed partial class StorageEngine private CompactionSnapshot CaptureCompactionSnapshot() { var fileSizeBytes = _pageFile.FileLengthBytes; - var pageCount = _pageFile.NextPageId; + var logicalPageCount = _pageFile.NextPageId; var pageSize = _pageFile.PageSize; + var pageCount = (uint)(fileSizeBytes / pageSize); var freePages = _pageFile.EnumerateFreePages(includeEmptyPages: true); - var freePageCount = freePages.Count; + var freePageSet = new HashSet(freePages); + + if (pageCount > logicalPageCount) + { + for (var pageId = logicalPageCount; pageId < pageCount; pageId++) + { + freePageSet.Add(pageId); + } + } + + var freePageCount = freePageSet.Count; long slottedFreeBytes = 0; - if (pageCount > 1) + if (logicalPageCount > 1) { var pageBuffer = new byte[pageSize]; - for (uint pageId = 1; pageId < pageCount; pageId++) + for (uint pageId = 1; pageId < logicalPageCount; pageId++) { _pageFile.ReadPage(pageId, pageBuffer); if (!TryReadSlottedFreeSpace(pageBuffer, out var availableFreeSpace)) @@ -805,7 +816,6 @@ public sealed partial class StorageEngine ? 0 : (totalFreeBytes * 100d) / fileSizeBytes; - var freePageSet = new HashSet(freePages); uint tailReclaimablePages = 0; for (var pageId = pageCount; pageId > 2; pageId--) {