using System.Globalization;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
///
/// Site-local SQLite source-of-truth for cached-operation tracking — the row
/// that Tracking.Status(TrackedOperationId) reads.
///
///
///
/// One row per ; lifecycle is
/// Submitted → Retrying → Delivered / Parked / Failed / Discarded; terminal
/// rows are purged after the configured retention window
/// (). Volume is bounded —
/// only cached calls produce rows, and only a handful of lifecycle events per
/// call — so we keep the implementation deliberately simple: a single owned
/// serialised behind a
/// (one async writer at a time) — simpler than a batched-channel pipeline given
/// the volume; the batched-channel audit-writer design is reserved for the
/// high-volume audit hot-path.
///
///
/// All mutations are idempotent / monotonic: is
/// INSERT OR IGNORE, filters out terminal
/// rows in the WHERE clause, and only
/// fires on rows that haven't terminated yet (first-write-wins). This makes the
/// store safe under the at-least-once semantics of the site→central telemetry
/// path.
///
///
public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable, IDisposable
{
// Writer state — one owned SqliteConnection serialised behind
// _writeGate. Readers do NOT share this connection or gate; see GetStatusAsync.
private readonly SqliteConnection _writeConnection;
private readonly SemaphoreSlim _writeGate = new(1, 1);
private readonly ILocalDb _localDb;
private readonly ILogger _logger;
// Dispose-once state shared by the sync Dispose and async
// DisposeAsync paths. Interlocked.Exchange is the race-safe primitive here —
// a plain bool can be flipped twice if Dispose() and DisposeAsync() are
// invoked concurrently (e.g. host shutdown bridging both). 0 = live,
// 1 = disposed. Read by other methods via Volatile.Read after the gate is
// taken; they raise ObjectDisposedException when set.
private int _disposeState;
///
/// Initializes the tracking store over the consolidated site database and applies the schema.
///
///
/// The consolidated site database. Every connection it hands out is already open and carries
/// the per-connection pragmas plus the zb_hlc_next() UDF that
/// OperationTracking's capture triggers call — which is exactly why the store no longer
/// opens its own from a connection string. A raw connection
/// would lack the UDF and every write to the replicated table would fail closed.
///
/// Logger for diagnostics.
///
/// no longer feeds this store — the
/// database location is LocalDb:Path. The option remains bound for the purge/retention
/// settings that share the class.
///
public OperationTrackingStore(
ILocalDb localDb,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(localDb);
ArgumentNullException.ThrowIfNull(logger);
_logger = logger;
_localDb = localDb;
// Already open — CreateConnection returns a live, pragma-configured,
// UDF-registered connection. Calling Open() on it again would throw.
_writeConnection = localDb.CreateConnection();
InitializeSchema();
}
// Schema lives in OperationTrackingSchema so the Host's AddZbLocalDb onReady
// callback can create this table in the consolidated site database before
// RegisterReplicated installs its capture triggers. In the host this call is
// therefore always a no-op — onReady runs while ILocalDb is being constructed,
// which is strictly before this constructor can receive it. It stays because it
// keeps a directly-constructed store (tests, tooling) self-sufficient, and the
// DDL is idempotent.
private void InitializeSchema() => OperationTrackingSchema.Apply(_writeConnection);
///
public async Task RecordEnqueueAsync(
TrackedOperationId id,
string kind,
string? targetSummary,
string? sourceInstanceId,
string? sourceScript,
string? sourceNode,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(kind);
await _writeGate.WaitAsync(ct).ConfigureAwait(false);
try
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture);
using var cmd = _writeConnection.CreateCommand();
// INSERT OR IGNORE: duplicate ids are no-ops (first-write-wins) —
// matches the at-least-once semantics the site emits under.
cmd.CommandText = """
INSERT OR IGNORE INTO OperationTracking (
TrackedOperationId, Kind, TargetSummary, Status,
RetryCount, LastError, HttpStatus,
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
SourceInstanceId, SourceScript, SourceNode
) VALUES (
$id, $kind, $targetSummary, $status,
0, NULL, NULL,
$now, $now, NULL,
$sourceInstanceId, $sourceScript, $sourceNode
);
""";
cmd.Parameters.AddWithValue("$id", id.ToString());
cmd.Parameters.AddWithValue("$kind", kind);
cmd.Parameters.AddWithValue("$targetSummary", (object?)targetSummary ?? DBNull.Value);
cmd.Parameters.AddWithValue("$status", "Submitted");
cmd.Parameters.AddWithValue("$now", now);
cmd.Parameters.AddWithValue("$sourceInstanceId", (object?)sourceInstanceId ?? DBNull.Value);
cmd.Parameters.AddWithValue("$sourceScript", (object?)sourceScript ?? DBNull.Value);
cmd.Parameters.AddWithValue("$sourceNode", (object?)sourceNode ?? DBNull.Value);
cmd.ExecuteNonQuery();
}
finally
{
_writeGate.Release();
}
}
///
public async Task RecordAttemptAsync(
TrackedOperationId id,
string status,
int retryCount,
string? lastError,
int? httpStatus,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(status);
await _writeGate.WaitAsync(ct).ConfigureAwait(false);
try
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture);
using var cmd = _writeConnection.CreateCommand();
// Terminal rows are immutable — the WHERE clause filters them out so
// late-arriving attempt telemetry never overwrites a resolved row.
cmd.CommandText = """
UPDATE OperationTracking
SET Status = $status,
RetryCount = $retryCount,
LastError = $lastError,
HttpStatus = $httpStatus,
UpdatedAtUtc = $now
WHERE TrackedOperationId = $id
AND TerminalAtUtc IS NULL;
""";
cmd.Parameters.AddWithValue("$id", id.ToString());
cmd.Parameters.AddWithValue("$status", status);
cmd.Parameters.AddWithValue("$retryCount", retryCount);
cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value);
cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value);
cmd.Parameters.AddWithValue("$now", now);
cmd.ExecuteNonQuery();
}
finally
{
_writeGate.Release();
}
}
///
public async Task RecordTerminalAsync(
TrackedOperationId id,
string status,
string? lastError,
int? httpStatus,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(status);
await _writeGate.WaitAsync(ct).ConfigureAwait(false);
try
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
var now = DateTime.UtcNow.ToString("o", CultureInfo.InvariantCulture);
using var cmd = _writeConnection.CreateCommand();
// First-write-wins on the terminal flip: only update rows that
// haven't already terminated.
cmd.CommandText = """
UPDATE OperationTracking
SET Status = $status,
LastError = $lastError,
HttpStatus = $httpStatus,
UpdatedAtUtc = $now,
TerminalAtUtc = $now
WHERE TrackedOperationId = $id
AND TerminalAtUtc IS NULL;
""";
cmd.Parameters.AddWithValue("$id", id.ToString());
cmd.Parameters.AddWithValue("$status", status);
cmd.Parameters.AddWithValue("$lastError", (object?)lastError ?? DBNull.Value);
cmd.Parameters.AddWithValue("$httpStatus", (object?)httpStatus ?? DBNull.Value);
cmd.Parameters.AddWithValue("$now", now);
cmd.ExecuteNonQuery();
}
finally
{
_writeGate.Release();
}
}
///
public async Task GetStatusAsync(
TrackedOperationId id,
CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
// Reads open a fresh, ungated connection so a long-running write doesn't
// block status queries. It comes from the same ILocalDb as the writer;
// SQLite handles cross-connection isolation natively (readers see a WAL
// snapshot). Mirrors the SiteStorageService precedent. Already open — do
// not call OpenAsync on it.
await using var readConnection = _localDb.CreateConnection();
await using var cmd = readConnection.CreateCommand();
cmd.CommandText = """
SELECT TrackedOperationId, Kind, TargetSummary, Status,
RetryCount, LastError, HttpStatus,
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc,
SourceInstanceId, SourceScript, SourceNode
FROM OperationTracking
WHERE TrackedOperationId = $id;
""";
cmd.Parameters.AddWithValue("$id", id.ToString());
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (!await reader.ReadAsync(ct).ConfigureAwait(false))
{
return null;
}
return new TrackingStatusSnapshot(
Id: TrackedOperationId.Parse(reader.GetString(0)),
Kind: reader.GetString(1),
TargetSummary: reader.IsDBNull(2) ? null : reader.GetString(2),
Status: reader.GetString(3),
RetryCount: reader.GetInt32(4),
LastError: reader.IsDBNull(5) ? null : reader.GetString(5),
HttpStatus: reader.IsDBNull(6) ? null : reader.GetInt32(6),
CreatedAtUtc: ParseUtc(reader.GetString(7)),
UpdatedAtUtc: ParseUtc(reader.GetString(8)),
TerminalAtUtc: reader.IsDBNull(9) ? null : ParseUtc(reader.GetString(9)),
SourceInstanceId: reader.IsDBNull(10) ? null : reader.GetString(10),
SourceScript: reader.IsDBNull(11) ? null : reader.GetString(11),
SourceNode: reader.IsDBNull(12) ? null : reader.GetString(12));
}
///
public async Task PurgeTerminalAsync(
DateTime olderThanUtc,
CancellationToken ct = default)
{
await _writeGate.WaitAsync(ct).ConfigureAwait(false);
try
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
using var cmd = _writeConnection.CreateCommand();
// Non-terminal rows (TerminalAtUtc IS NULL) are kept regardless of
// age — the operation is still in flight.
cmd.CommandText = """
DELETE FROM OperationTracking
WHERE TerminalAtUtc IS NOT NULL
AND TerminalAtUtc < $threshold;
""";
cmd.Parameters.AddWithValue(
"$threshold",
olderThanUtc.ToString("o", CultureInfo.InvariantCulture));
cmd.ExecuteNonQuery();
}
finally
{
_writeGate.Release();
}
}
///
public async Task> ReadChangedSinceAsync(
DateTime sinceUtc,
int batchSize,
string? afterId = null,
CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposeState) != 0, this);
// Like GetStatusAsync, the reconciliation pull opens a
// fresh, ungated read connection so a long-running write never blocks
// central's PullSiteCalls. The query is a bounded, ordered scan served by
// the standalone IX_OperationTracking_UpdatedAt index — UpdatedAtUtc is
// the cursor. (The composite (Status, UpdatedAtUtc) index cannot satisfy a
// status-less UpdatedAtUtc range scan; this dedicated index does.)
// Already open — do not call OpenAsync on it.
await using var readConnection = _localDb.CreateConnection();
await using var cmd = readConnection.CreateCommand();
// Composite (UpdatedAtUtc, TrackedOperationId) keyset. Ordering
// is ALWAYS deterministic (UpdatedAtUtc ASC, TrackedOperationId ASC) so the
// cursor is well-defined even when many rows share one UpdatedAtUtc.
// • afterId present → strict keyset: skip everything up to and including
// (sinceUtc, afterId). Without this a page fully inside one UpdatedAtUtc
// instant re-reads forever (the plain >= resume never advances past the
// tie). TrackedOperationId is the "D" GUID form on both sides, so the
// TEXT comparison is a stable total order.
// • afterId null/empty → legacy inclusive >= sinceUtc (an older central
// that does not send a cursor is byte-for-byte unaffected).
var useKeyset = !string.IsNullOrEmpty(afterId);
var predicate = useKeyset
? "UpdatedAtUtc > $since OR (UpdatedAtUtc = $since AND TrackedOperationId > $afterId)"
: "UpdatedAtUtc >= $since";
cmd.CommandText = $"""
SELECT TrackedOperationId, Kind, TargetSummary, Status,
RetryCount, LastError, HttpStatus,
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, SourceNode
FROM OperationTracking
WHERE {predicate}
ORDER BY UpdatedAtUtc ASC, TrackedOperationId ASC
LIMIT $batchSize;
""";
// Force UTC kind before formatting so the cursor's "o" text matches the
// 'Z'-suffixed round-trip form the write path persists (DateTime.UtcNow
// .ToString("o")). A first-cycle DateTime.MinValue arrives Unspecified —
// without this its "o" rendering would lack the 'Z', and the SQLite text
// compare against 'Z'-suffixed stored values would be subtly inconsistent.
var sinceText = DateTime
.SpecifyKind(sinceUtc, DateTimeKind.Utc)
.ToString("o", CultureInfo.InvariantCulture);
cmd.Parameters.AddWithValue("$since", sinceText);
cmd.Parameters.AddWithValue("$batchSize", batchSize);
if (useKeyset)
{
cmd.Parameters.AddWithValue("$afterId", afterId!);
}
var rows = new List();
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
var kind = reader.GetString(1);
rows.Add(new SiteCallOperational(
TrackedOperationId: TrackedOperationId.Parse(reader.GetString(0)),
Channel: KindToChannel(kind),
Target: reader.IsDBNull(2) ? string.Empty : reader.GetString(2),
// The site id is not a tracking-store column; the central client
// re-stamps SourceSite from the siteId it dialed.
SourceSite: string.Empty,
SourceNode: reader.IsDBNull(10) ? null : reader.GetString(10),
Status: reader.GetString(3),
RetryCount: reader.GetInt32(4),
LastError: reader.IsDBNull(5) ? null : reader.GetString(5),
HttpStatus: reader.IsDBNull(6) ? null : reader.GetInt32(6),
CreatedAtUtc: ParseUtc(reader.GetString(7)),
UpdatedAtUtc: ParseUtc(reader.GetString(8)),
TerminalAtUtc: reader.IsDBNull(9) ? null : ParseUtc(reader.GetString(9))));
}
return rows;
}
// Cached-call Kind → SiteCalls Channel. Only ApiCallCached / DbWriteCached
// ever reach the tracking store (RecordEnqueueAsync is the cached-call
// entry point); DbWriteCached maps to DbOutbound, everything else to the
// ApiOutbound default. Mirrors CachedCallLifecycleBridge's channel handling.
private static string KindToChannel(string kind) => kind switch
{
nameof(Commons.Types.Enums.AuditKind.DbWriteCached) => nameof(Commons.Types.Enums.AuditChannel.DbOutbound),
_ => nameof(Commons.Types.Enums.AuditChannel.ApiOutbound),
};
private static DateTime ParseUtc(string raw)
{
return DateTime.Parse(
raw,
CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind);
}
///
/// Synchronously disposes the tracking store and its SQLite connection.
///
///
/// This path does NOT bridge to async via
/// .AsTask().GetAwaiter().GetResult(). Sync-over-async on a SemaphoreSlim
/// can deadlock when invoked from a non-reentrant SyncContext (e.g. host
/// shutdown continuations observed on the host sync context). In-flight writes
/// at the moment of will fail their next operation
/// against the disposed connection with —
/// the caller's responsibility is to ensure no concurrent operations during
/// the synchronous dispose. Use if you need to
/// drain in-flight writes before close.
///
public void Dispose()
{
if (Interlocked.Exchange(ref _disposeState, 1) != 0)
{
return;
}
_writeConnection.Dispose();
_writeGate.Dispose();
GC.SuppressFinalize(this);
}
///
/// Asynchronously disposes the tracking store and its SQLite connection.
/// Drains in-flight writes by acquiring the write gate before closing the
/// connection, so a write currently executing a SqliteCommand completes
/// before the connection is freed.
///
/// A that completes when all resources have been released.
public async ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref _disposeState, 1) != 0)
{
return;
}
// Drain any in-flight write by taking the write gate. Past this point
// no new write can acquire the gate because _disposeState is set, so
// the next ThrowIf check in each writer raises ObjectDisposedException.
try
{
await _writeGate.WaitAsync().ConfigureAwait(false);
}
catch (ObjectDisposedException)
{
// Race with another disposer that already disposed the gate — the
// _disposeState exchange above should prevent this, but be defensive.
}
try
{
_writeConnection.Dispose();
}
finally
{
try { _writeGate.Release(); } catch (ObjectDisposedException) { }
_writeGate.Dispose();
}
GC.SuppressFinalize(this);
}
}