using System.Globalization; using System.Threading.Channels; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces; using ZB.MOM.WW.ScadaBridge.Commons.Types; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking; /// /// Site-local SQLite source-of-truth for cached-operation tracking — the row /// that Tracking.Status(TrackedOperationId) reads. /// /// /// /// One row per ; lifecycle is /// Submitted → Retrying → Delivered / Parked / Failed / Discarded; terminal /// rows are purged after the configured retention window /// (). Volume is bounded — /// only cached calls produce rows, and only a handful of lifecycle events per /// call — but every one of those rows used to be its own autocommit transaction /// awaited on the script thread (through CachedCallTelemetryForwarder), /// and a WAL commit on the consolidated LocalDb file costs a fsync (~30–50 ms on a /// Docker volume): a single Database.CachedWrite paid three of them, ~108 ms /// of a 176 ms script run measured on the rig. Since 2026-09-06 the store is a /// single-writer queue: / /// / enqueue an /// ordered work item and return, and one background pump applies each drained batch /// in one transaction on the single owned . Reads /// (, ) wait for every /// write queued before them — a barrier through the same queue — so /// Tracking.Status(id) keeps read-your-writes and the telemetry actor's status /// join never sees a stale row. rides the queue too /// and returns after the purge is applied. Design memo: /// docs/plans/2026-09-06-tracking-store-async-writes-design.md. /// /// /// Accepted trade: a row queued but not yet applied is lost on a hard crash (the /// pump's cadence, milliseconds). The tracking row then stays at its previous state /// until reconciled; the S&F buffer and audit rows are unaffected. The queue is /// bounded () with wait-mode backpressure — tracking is /// the status source of truth, so it never drops. /// /// /// All mutations are idempotent / monotonic: is /// INSERT OR IGNORE, filters out terminal /// rows in the WHERE clause, and only /// fires on rows that haven't terminated yet (first-write-wins). This makes the /// store safe under the at-least-once semantics of the site→central telemetry /// path. /// /// public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable, IDisposable { /// Queue bound; past it Record*Async waits (never drops). internal const int MaxQueuedWrites = 8192; /// Most work items applied in one transaction. internal const int MaxBatchSize = 256; // Writer state — one owned SqliteConnection, touched ONLY by the pump (and by // DisposeAsync once the pump has finished), serialised behind _writeGate. // Readers do NOT share this connection or gate; see GetStatusAsync. private readonly SqliteConnection _writeConnection; private readonly SemaphoreSlim _writeGate = new(1, 1); private readonly ILocalDb _localDb; private readonly ILogger _logger; // The single-writer queue. One reader (the pump), many writers (script threads, // the S&F lifecycle bridge, the purge loop). FIFO is what keeps // enqueue → attempt → terminal applying in the order they were recorded. private readonly Channel _queue = Channel.CreateBounded( new BoundedChannelOptions(MaxQueuedWrites) { SingleReader = true, SingleWriter = false, FullMode = BoundedChannelFullMode.Wait, }); private readonly Task _pump; // Items queued and not yet applied. A read that observes 0 has nothing to wait // for and skips the barrier; otherwise it queues one and waits its turn. private int _pending; /// /// One queued mutation. runs on the pump inside the batch /// transaction; (when present) is completed after the /// item is durably applied, or faulted when it failed — barriers and the purge /// use it, plain lifecycle writes do not wait. /// private sealed record WorkItem( string Description, Action? Apply, TaskCompletionSource? Completion); // Dispose-once state shared by the sync Dispose and async // DisposeAsync paths. Interlocked.Exchange is the race-safe primitive here — // a plain bool can be flipped twice if Dispose() and DisposeAsync() are // invoked concurrently (e.g. host shutdown bridging both). 0 = live, // 1 = disposed. Read by other methods via Volatile.Read after the gate is // taken; they raise ObjectDisposedException when set. private int _disposeState; /// /// Initializes the tracking store over the consolidated site database and applies the schema. /// /// /// The consolidated site database. Every connection it hands out is already open and carries /// the per-connection pragmas plus the zb_hlc_next() UDF that /// OperationTracking's capture triggers call — which is exactly why the store no longer /// opens its own from a connection string. A raw connection /// would lack the UDF and every write to the replicated table would fail closed. /// /// Logger for diagnostics. /// /// no longer feeds this store — the /// database location is LocalDb:Path. The option remains bound for the purge/retention /// settings that share the class. /// public OperationTrackingStore( ILocalDb localDb, ILogger logger) { ArgumentNullException.ThrowIfNull(localDb); ArgumentNullException.ThrowIfNull(logger); _logger = logger; _localDb = localDb; // Already open — CreateConnection returns a live, pragma-configured, // UDF-registered connection. Calling Open() on it again would throw. _writeConnection = localDb.CreateConnection(); InitializeSchema(); _pump = Task.Run(PumpAsync); } // Schema lives in OperationTrackingSchema so the Host's AddZbLocalDb onReady // callback can create this table in the consolidated site database before // RegisterReplicated installs its capture triggers. In the host this call is // therefore always a no-op — onReady runs while ILocalDb is being constructed, // which is strictly before this constructor can receive it. It stays because it // keeps a directly-constructed store (tests, tooling) self-sufficient, and the // DDL is idempotent. private void InitializeSchema() => OperationTrackingSchema.Apply(_writeConnection); /// public async Task RecordEnqueueAsync( TrackedOperationId id, string kind, string? targetSummary, string? sourceInstanceId, string? sourceScript, string? sourceNode, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(kind); // Stamped when RECORDED, not when the batch lands. var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); var idText = id.ToString(); await EnqueueAsync($"enqueue {idText}", (conn, tx) => { using var cmd = conn.CreateCommand(); cmd.Transaction = tx; // INSERT OR IGNORE: duplicate ids are no-ops (first-write-wins) — // matches the at-least-once semantics the site emits under. cmd.CommandText = """ INSERT OR IGNORE INTO OperationTracking ( TrackedOperationId, Kind, TargetSummary, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, SourceInstanceId, SourceScript, SourceNode ) VALUES ( $id, $kind, $targetSummary, $status, 0, NULL, NULL, $now, $now, NULL, $sourceInstanceId, $sourceScript, $sourceNode ); """; cmd.Parameters.AddWithValue("$id", idText); cmd.Parameters.AddWithValue("$kind", kind); cmd.Parameters.AddWithValue("$targetSummary", (object?)targetSummary ?? DBNull.Value); cmd.Parameters.AddWithValue("$status", "Submitted"); cmd.Parameters.AddWithValue("$now", now); cmd.Parameters.AddWithValue("$sourceInstanceId", (object?)sourceInstanceId ?? DBNull.Value); cmd.Parameters.AddWithValue("$sourceScript", (object?)sourceScript ?? DBNull.Value); cmd.Parameters.AddWithValue("$sourceNode", (object?)sourceNode ?? DBNull.Value); cmd.ExecuteNonQuery(); }, ct).ConfigureAwait(false); } /// public async Task RecordAttemptAsync( TrackedOperationId id, string status, int retryCount, string? lastError, int? httpStatus, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(status); var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); var idText = id.ToString(); await EnqueueAsync($"attempt {idText}", (conn, tx) => { using var cmd = conn.CreateCommand(); cmd.Transaction = tx; // Terminal rows are immutable — the WHERE clause filters them out so // late-arriving attempt telemetry never overwrites a resolved row. cmd.CommandText = """ UPDATE OperationTracking SET Status = $status, RetryCount = $retryCount, LastError = $lastError, HttpStatus = $httpStatus, UpdatedAtUtc = $now WHERE TrackedOperationId = $id AND TerminalAtUtc IS NULL; """; cmd.Parameters.AddWithValue("$id", idText); cmd.Parameters.AddWithValue("$status", status); cmd.Parameters.AddWithValue("$retryCount", retryCount); cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value); cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value); cmd.Parameters.AddWithValue("$now", now); cmd.ExecuteNonQuery(); }, ct).ConfigureAwait(false); } /// public async Task RecordTerminalAsync( TrackedOperationId id, string status, string? lastError, int? httpStatus, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(status); var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture); var idText = id.ToString(); await EnqueueAsync($"terminal {idText}", (conn, tx) => { using var cmd = conn.CreateCommand(); cmd.Transaction = tx; // First-write-wins on the terminal flip: only update rows that // haven't already terminated. cmd.CommandText = """ UPDATE OperationTracking SET Status = $status, LastError = $lastError, HttpStatus = $httpStatus, UpdatedAtUtc = $now, TerminalAtUtc = $now WHERE TrackedOperationId = $id AND TerminalAtUtc IS NULL; """; cmd.Parameters.AddWithValue("$id", idText); cmd.Parameters.AddWithValue("$status", status); cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value); cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value); cmd.Parameters.AddWithValue("$now", now); cmd.ExecuteNonQuery(); }, ct).ConfigureAwait(false); } // ── Queue plumbing ───────────────────────────────────────────────────── /// /// Completes once every write queued before this call has been applied. Reads /// take this barrier implicitly; it is public for callers that go around the /// store to the file (tests, tooling) and for anyone who wants "durable now". /// public Task FlushAsync(CancellationToken ct = default) => WaitForQueuedWritesAsync(ct); /// Queues a lifecycle write and returns once it is queued (not applied). private Task EnqueueAsync(string description, Action apply, CancellationToken ct) => EnqueueAsync(new WorkItem(description, apply, Completion: null), ct); /// Queues a work item; a completion-bearing item is awaited by its owner separately. private async Task EnqueueAsync(WorkItem item, CancellationToken ct) { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this); Interlocked.Increment(ref _pending); try { await _queue.Writer.WriteAsync(item, ct).ConfigureAwait(false); } catch (ChannelClosedException) { Interlocked.Decrement(ref _pending); throw new ObjectDisposedException(GetType().FullName); } catch { Interlocked.Decrement(ref _pending); throw; } } /// /// Read barrier: completes once every write queued before this call has been /// applied. Skipped when nothing is pending. A disposed store lets the read /// proceed to its own ObjectDisposed check. /// private async Task WaitForQueuedWritesAsync(CancellationToken ct) { if (Volatile.Read(ref _pending) == 0) return; var barrier = new WorkItem("barrier", Apply: null, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); try { await EnqueueAsync(barrier, ct).ConfigureAwait(false); } catch (ObjectDisposedException) { return; } await barrier.Completion!.Task.WaitAsync(ct).ConfigureAwait(false); } /// /// The single writer. Drains the queue in batches, applies each batch in one /// transaction (one fsync for the whole batch), and falls back to one row at a /// time when a batch fails so a poison row never takes its neighbours down. /// Never throws — it runs for the store's lifetime and ends when the queue is /// completed by disposal. /// private async Task PumpAsync() { var reader = _queue.Reader; var batch = new List(MaxBatchSize); try { while (await reader.WaitToReadAsync().ConfigureAwait(false)) { batch.Clear(); while (batch.Count < MaxBatchSize && reader.TryRead(out var item)) batch.Add(item); if (batch.Count == 0) continue; await _writeGate.WaitAsync().ConfigureAwait(false); try { ApplyBatch(batch); } finally { _writeGate.Release(); } } } catch (Exception ex) { // Only the gate itself can throw here (disposed under us on the sync // Dispose path). Fail whatever is still queued so no awaiter hangs. _logger.LogWarning(ex, "OperationTrackingStore: write pump stopped; failing queued tracking writes"); foreach (var item in batch) Fail(item, ex); while (reader.TryRead(out var left)) Fail(left, ex); } } private void ApplyBatch(List batch) { if (Volatile.Read(ref _disposeState) != 0 && _writeConnection.State != System.Data.ConnectionState.Open) { // Sync Dispose() closed the connection while items were queued — the // documented "in-flight writes fail" semantics of that path. var disposed = new ObjectDisposedException(GetType().FullName); foreach (var item in batch) Fail(item, disposed); return; } try { using var tx = _writeConnection.BeginTransaction(); foreach (var item in batch) item.Apply?.Invoke(_writeConnection, tx); tx.Commit(); foreach (var item in batch) Complete(item); return; } catch (Exception ex) { _logger.LogWarning(ex, "OperationTrackingStore: batch of {Count} tracking writes failed; re-applying one row at a time", batch.Count); } // Per-row fallback: keep the good rows, log and fail only the bad one. foreach (var item in batch) { try { using var tx = _writeConnection.BeginTransaction(); item.Apply?.Invoke(_writeConnection, tx); tx.Commit(); Complete(item); } catch (Exception ex) { _logger.LogWarning(ex, "OperationTrackingStore: tracking write failed ({Description})", item.Description); Fail(item, ex); } } } private void Complete(WorkItem item) { Interlocked.Decrement(ref _pending); item.Completion?.TrySetResult(); } private void Fail(WorkItem item, Exception ex) { Interlocked.Decrement(ref _pending); item.Completion?.TrySetException(ex); } /// public async Task GetStatusAsync( TrackedOperationId id, CancellationToken ct = default) { // Read-your-writes: drain what was queued before this call (no-op when idle). await WaitForQueuedWritesAsync(ct).ConfigureAwait(false); ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this); // Reads open a fresh, ungated connection so a long-running write doesn't // block status queries. It comes from the same ILocalDb as the writer; // SQLite handles cross-connection isolation natively (readers see a WAL // snapshot). Mirrors the SiteStorageService precedent. Already open — do // not call OpenAsync on it. await using var readConnection = _localDb.CreateConnection(); await using var cmd = readConnection.CreateCommand(); cmd.CommandText = """ SELECT TrackedOperationId, Kind, TargetSummary, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, SourceInstanceId, SourceScript, SourceNode FROM OperationTracking WHERE TrackedOperationId = $id; """; cmd.Parameters.AddWithValue("$id", id.ToString()); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); if (!await reader.ReadAsync(ct).ConfigureAwait(false)) { return null; } return new TrackingStatusSnapshot( Id: TrackedOperationId.Parse(reader.GetString(0)), Kind: reader.GetString(1), TargetSummary: reader.IsDBNull(2) ? null : reader.GetString(2), Status: reader.GetString(3), RetryCount: reader.GetInt32(4), LastError: reader.IsDBNull(5) ? null : reader.GetString(5), HttpStatus: reader.IsDBNull(6) ? null : reader.GetInt32(6), CreatedAtUtc: ParseUtc(reader.GetString(7)), UpdatedAtUtc: ParseUtc(reader.GetString(8)), TerminalAtUtc: reader.IsDBNull(9) ? null : ParseUtc(reader.GetString(9)), SourceInstanceId: reader.IsDBNull(10) ? null : reader.GetString(10), SourceScript: reader.IsDBNull(11) ? null : reader.GetString(11), SourceNode: reader.IsDBNull(12) ? null : reader.GetString(12)); } /// public async Task PurgeTerminalAsync( DateTime olderThanUtc, CancellationToken ct = default) { var threshold = olderThanUtc.ToString("o", CultureInfo.InvariantCulture); // The purge rides the queue (FIFO with the lifecycle writes) and, unlike // them, is awaited to completion so the caller's "purged" means purged. var purge = new WorkItem("purge", (conn, tx) => { using var cmd = conn.CreateCommand(); cmd.Transaction = tx; // Non-terminal rows (TerminalAtUtc IS NULL) are kept regardless of // age — the operation is still in flight. cmd.CommandText = """ DELETE FROM OperationTracking WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < $threshold; """; cmd.Parameters.AddWithValue("$threshold", threshold); cmd.ExecuteNonQuery(); }, new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); await EnqueueAsync(purge, ct).ConfigureAwait(false); await purge.Completion!.Task.WaitAsync(ct).ConfigureAwait(false); } /// public async Task> ReadChangedSinceAsync( DateTime sinceUtc, int batchSize, string? afterId = null, CancellationToken ct = default) { // Read-your-writes for central's reconciliation pull, same as GetStatusAsync. await WaitForQueuedWritesAsync(ct).ConfigureAwait(false); ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this); // Like GetStatusAsync, the reconciliation pull opens a // fresh, ungated read connection so a long-running write never blocks // central's PullSiteCalls. The query is a bounded, ordered scan served by // the standalone IX_OperationTracking_UpdatedAt index — UpdatedAtUtc is // the cursor. (The composite (Status, UpdatedAtUtc) index cannot satisfy a // status-less UpdatedAtUtc range scan; this dedicated index does.) // Already open — do not call OpenAsync on it. await using var readConnection = _localDb.CreateConnection(); await using var cmd = readConnection.CreateCommand(); // Composite (UpdatedAtUtc, TrackedOperationId) keyset. Ordering // is ALWAYS deterministic (UpdatedAtUtc ASC, TrackedOperationId ASC) so the // cursor is well-defined even when many rows share one UpdatedAtUtc. // • afterId present → strict keyset: skip everything up to and including // (sinceUtc, afterId). Without this a page fully inside one UpdatedAtUtc // instant re-reads forever (the plain >= resume never advances past the // tie). TrackedOperationId is the "D" GUID form on both sides, so the // TEXT comparison is a stable total order. // • afterId null/empty → legacy inclusive >= sinceUtc (an older central // that does not send a cursor is byte-for-byte unaffected). var useKeyset = !string.IsNullOrEmpty(afterId); var predicate = useKeyset ? "UpdatedAtUtc > $since OR (UpdatedAtUtc = $since AND TrackedOperationId > $afterId)" : "UpdatedAtUtc >= $since"; cmd.CommandText = $""" SELECT TrackedOperationId, Kind, TargetSummary, Status, RetryCount, LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, SourceNode FROM OperationTracking WHERE {predicate} ORDER BY UpdatedAtUtc ASC, TrackedOperationId ASC LIMIT $batchSize; """; // Force UTC kind before formatting so the cursor's "o" text matches the // 'Z'-suffixed round-trip form the write path persists (DateTime.UtcNow // .ToString("o")). A first-cycle DateTime.MinValue arrives Unspecified — // without this its "o" rendering would lack the 'Z', and the SQLite text // compare against 'Z'-suffixed stored values would be subtly inconsistent. var sinceText = DateTime .SpecifyKind(sinceUtc, DateTimeKind.Utc) .ToString("o", CultureInfo.InvariantCulture); cmd.Parameters.AddWithValue("$since", sinceText); cmd.Parameters.AddWithValue("$batchSize", batchSize); if (useKeyset) { cmd.Parameters.AddWithValue("$afterId", afterId!); } var rows = new List(); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { var kind = reader.GetString(1); rows.Add(new SiteCallOperational( TrackedOperationId: TrackedOperationId.Parse(reader.GetString(0)), Channel: KindToChannel(kind), Target: reader.IsDBNull(2) ? string.Empty : reader.GetString(2), // The site id is not a tracking-store column; the central client // re-stamps SourceSite from the siteId it dialed. SourceSite: string.Empty, SourceNode: reader.IsDBNull(10) ? null : reader.GetString(10), Status: reader.GetString(3), RetryCount: reader.GetInt32(4), LastError: reader.IsDBNull(5) ? null : reader.GetString(5), HttpStatus: reader.IsDBNull(6) ? null : reader.GetInt32(6), CreatedAtUtc: ParseUtc(reader.GetString(7)), UpdatedAtUtc: ParseUtc(reader.GetString(8)), TerminalAtUtc: reader.IsDBNull(9) ? null : ParseUtc(reader.GetString(9)))); } return rows; } // Cached-call Kind → SiteCalls Channel. Only ApiCallCached / DbWriteCached // ever reach the tracking store (RecordEnqueueAsync is the cached-call // entry point); DbWriteCached maps to DbOutbound, everything else to the // ApiOutbound default. Mirrors CachedCallLifecycleBridge's channel handling. private static string KindToChannel(string kind) => kind switch { nameof(Commons.Types.Enums.AuditKind.DbWriteCached) => nameof(Commons.Types.Enums.AuditChannel.DbOutbound), _ => nameof(Commons.Types.Enums.AuditChannel.ApiOutbound), }; private static DateTime ParseUtc(string raw) { return DateTime.Parse( raw, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); } /// /// Synchronously disposes the tracking store and its SQLite connection. /// /// /// This path does NOT bridge to async via /// .AsTask().GetAwaiter().GetResult(). Sync-over-async on a SemaphoreSlim /// can deadlock when invoked from a non-reentrant SyncContext (e.g. host /// shutdown continuations observed on the host sync context). In-flight writes /// at the moment of will fail their next operation /// against the disposed connection with — /// the caller's responsibility is to ensure no concurrent operations during /// the synchronous dispose. Use if you need to /// drain in-flight writes before close. /// public void Dispose() { if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } // Stop accepting work; whatever is still queued fails on the pump with // ObjectDisposedException (see ApplyBatch) — this path does not drain. _queue.Writer.TryComplete(); _writeConnection.Dispose(); _writeGate.Dispose(); GC.SuppressFinalize(this); } /// /// Asynchronously disposes the tracking store and its SQLite connection. /// Drains the write queue first (completes it and awaits the pump), so every /// write that was queued before disposal is applied before the connection is /// freed. This is the path host shutdown must use. /// /// A that completes when all resources have been released. public async ValueTask DisposeAsync() { if (Interlocked.Exchange(ref _disposeState, 1) != 0) { return; } // Drain: no new item can be queued past this point (_disposeState is set // and the writer is completed), so awaiting the pump means every queued // write has been applied. Then take the gate so nothing else holds the // connection while it closes. _queue.Writer.TryComplete(); try { await _pump.ConfigureAwait(false); } catch (Exception ex) { _logger.LogWarning(ex, "OperationTrackingStore: write pump faulted during DisposeAsync"); } try { await _writeGate.WaitAsync().ConfigureAwait(false); } catch (ObjectDisposedException) { // Race with another disposer that already disposed the gate — the // _disposeState exchange above should prevent this, but be defensive. } try { _writeConnection.Dispose(); } finally { try { _writeGate.Release(); } catch (ObjectDisposedException) { } _writeGate.Dispose(); } GC.SuppressFinalize(this); } }