using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; using ZB.MOM.WW.LocalDb; using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; namespace ZB.MOM.WW.ScadaBridge.StoreAndForward; /// /// SQLite persistence layer for store-and-forward messages. /// Uses direct Microsoft.Data.Sqlite (not EF Core) for lightweight site-side storage. /// No max buffer size per design decision. /// public class StoreAndForwardStorage { private readonly ILocalDb _localDb; private readonly ILogger _logger; /// /// Initializes the store over the consolidated site database and applies the schema. /// /// /// The consolidated site database. Every connection it hands out is already open and carries /// the per-connection pragmas (including busy_timeout) plus the zb_hlc_next() /// UDF that sf_messages' capture triggers call — which is exactly why the store no /// longer opens its own from a connection string. A raw /// connection would lack the UDF and every write to the replicated table would fail closed. /// /// Logger for diagnostics. public StoreAndForwardStorage(ILocalDb localDb, ILogger logger) { ArgumentNullException.ThrowIfNull(localDb); ArgumentNullException.ThrowIfNull(logger); _localDb = localDb; _logger = logger; } /// /// Creates the sf_messages table if it does not exist. /// /// A task that represents the asynchronous operation. public Task InitializeAsync() { // No directory creation and no journal-mode pragma here any more: LocalDb owns // the file (it creates the directory) and sets WAL plus the per-connection // pragmas on every connection it hands out. using var connection = OpenConnection(); // The DDL itself lives in StoreAndForwardSchema so the Host can apply it to a // LocalDb-managed connection before RegisterReplicated installs the capture // triggers. The store still calls it, so a directly-constructed store (tests, // tooling) remains self-sufficient. StoreAndForwardSchema.Apply(connection); _logger.LogInformation("Store-and-forward SQLite storage initialized"); return Task.CompletedTask; } /// /// CreateConnection returns an ALREADY-OPEN, pragma-configured connection with the /// zb_hlc_next() UDF registered. Calling OpenAsync on it throws, and a raw /// would lack the UDF, making every capture trigger fail /// closed. /// private SqliteConnection OpenConnection() => _localDb.CreateConnection(); /// /// INSERT statement for a full message row. Shared by /// and ; bind with /// so the column list and the parameters never drift apart. /// private const string InsertMessageSql = @" INSERT INTO sf_messages (id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, last_attempt_at_ms, status, last_error, origin_instance, execution_id, source_script, parent_execution_id) VALUES (@id, @category, @target, @payload, @retryCount, @maxRetries, @retryIntervalMs, @createdAt, @lastAttempt, @lastAttemptMs, @status, @lastError, @origin, @executionId, @sourceScript, @parentExecutionId)"; /// /// Binds every column parameter for a full message row (used by /// ). GUID ids are stored in canonical "D" string /// form and the epoch-ms sibling of last_attempt_at is derived here so a single /// place owns the mapping. /// private static void BindMessageParameters(SqliteCommand cmd, StoreAndForwardMessage message) { cmd.Parameters.AddWithValue("@id", message.Id); cmd.Parameters.AddWithValue("@category", (int)message.Category); cmd.Parameters.AddWithValue("@target", message.Target); cmd.Parameters.AddWithValue("@payload", message.PayloadJson); cmd.Parameters.AddWithValue("@retryCount", message.RetryCount); cmd.Parameters.AddWithValue("@maxRetries", message.MaxRetries); cmd.Parameters.AddWithValue("@retryIntervalMs", message.RetryIntervalMs); cmd.Parameters.AddWithValue("@createdAt", message.CreatedAt.ToString("O")); cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue ? message.LastAttemptAt.Value.ToString("O") : DBNull.Value); cmd.Parameters.AddWithValue("@lastAttemptMs", (object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value); cmd.Parameters.AddWithValue("@status", (int)message.Status); cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value); cmd.Parameters.AddWithValue("@origin", (object?)message.OriginInstanceName ?? DBNull.Value); // GUID ids stored as canonical "D" strings; null when not threaded. cmd.Parameters.AddWithValue("@executionId", message.ExecutionId.HasValue ? message.ExecutionId.Value.ToString("D") : DBNull.Value); cmd.Parameters.AddWithValue("@sourceScript", (object?)message.SourceScript ?? DBNull.Value); cmd.Parameters.AddWithValue("@parentExecutionId", message.ParentExecutionId.HasValue ? message.ParentExecutionId.Value.ToString("D") : DBNull.Value); } /// /// Enqueues a new message with Pending status. /// /// The message to enqueue. /// A task that represents the asynchronous operation. public async Task EnqueueAsync(StoreAndForwardMessage message) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = InsertMessageSql; BindMessageParameters(cmd, message); await cmd.ExecuteNonQueryAsync(); } /// /// Returns every buffered message regardless of status, oldest-first, up to /// rows, plus a flag indicating the buffer holds more /// than that. Anti-entropy groundwork: the active node snapshots its buffer for /// a standby resync. One extra row is fetched (limit + 1) purely to /// compute the truncation flag without a second COUNT round-trip. /// /// Maximum rows to return; must be positive. /// The oldest-first page (at most rows) and whether more rows exist beyond it. public async Task<(List Messages, bool Truncated)> GetAllMessagesAsync(int limit) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" SELECT id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance, execution_id, source_script, parent_execution_id FROM sf_messages ORDER BY created_at ASC LIMIT @limitPlusOne"; cmd.Parameters.AddWithValue("@limitPlusOne", limit + 1); var rows = await ReadMessagesAsync(cmd); var truncated = rows.Count > limit; return (rows.Take(limit).ToList(), truncated); } /// /// Inserts a message, or updates every mutable column in place if a row with the /// same id already exists (ON CONFLICT(id) DO UPDATE). Used by the standby /// node when applying a replicated Add/Park/Requeue: a Park/Requeue whose original /// Add was lost self-heals (the full message rides in the operation), and a /// duplicate Add (e.g. after an anti-entropy resync) applies newest-wins instead of /// violating the primary key. The original created_at is preserved — it /// orders the retry sweep and must never be overwritten on update. /// /// The message to insert or update. /// A task that represents the asynchronous operation. public async Task UpsertMessageAsync(StoreAndForwardMessage message) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" INSERT INTO sf_messages (id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, last_attempt_at_ms, status, last_error, origin_instance, execution_id, source_script, parent_execution_id) VALUES (@id, @category, @target, @payload, @retryCount, @maxRetries, @retryIntervalMs, @createdAt, @lastAttempt, @lastAttemptMs, @status, @lastError, @origin, @executionId, @sourceScript, @parentExecutionId) ON CONFLICT(id) DO UPDATE SET category = excluded.category, target = excluded.target, payload_json = excluded.payload_json, retry_count = excluded.retry_count, max_retries = excluded.max_retries, retry_interval_ms = excluded.retry_interval_ms, last_attempt_at = excluded.last_attempt_at, last_attempt_at_ms = excluded.last_attempt_at_ms, status = excluded.status, last_error = excluded.last_error, origin_instance = excluded.origin_instance, execution_id = excluded.execution_id, source_script = excluded.source_script, parent_execution_id = excluded.parent_execution_id"; cmd.Parameters.AddWithValue("@id", message.Id); cmd.Parameters.AddWithValue("@category", (int)message.Category); cmd.Parameters.AddWithValue("@target", message.Target); cmd.Parameters.AddWithValue("@payload", message.PayloadJson); cmd.Parameters.AddWithValue("@retryCount", message.RetryCount); cmd.Parameters.AddWithValue("@maxRetries", message.MaxRetries); cmd.Parameters.AddWithValue("@retryIntervalMs", message.RetryIntervalMs); cmd.Parameters.AddWithValue("@createdAt", message.CreatedAt.ToString("O")); cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue ? message.LastAttemptAt.Value.ToString("O") : DBNull.Value); cmd.Parameters.AddWithValue("@lastAttemptMs", (object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value); cmd.Parameters.AddWithValue("@status", (int)message.Status); cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value); cmd.Parameters.AddWithValue("@origin", (object?)message.OriginInstanceName ?? DBNull.Value); cmd.Parameters.AddWithValue("@executionId", message.ExecutionId.HasValue ? message.ExecutionId.Value.ToString("D") : DBNull.Value); cmd.Parameters.AddWithValue("@sourceScript", (object?)message.SourceScript ?? DBNull.Value); cmd.Parameters.AddWithValue("@parentExecutionId", message.ParentExecutionId.HasValue ? message.ParentExecutionId.Value.ToString("D") : DBNull.Value); await cmd.ExecuteNonQueryAsync(); } /// /// Gets all messages that are due for retry (Pending status, last attempt older than retry interval). /// /// Maximum number of messages to return, or 0 for no limit. /// A task that resolves to the list of messages due for retry, ordered by creation time ascending. public async Task> GetMessagesForRetryAsync(int limit = 0) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" SELECT id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance, execution_id, source_script, parent_execution_id FROM sf_messages WHERE status = @pending AND (last_attempt_at_ms IS NULL OR retry_interval_ms = 0 OR (@nowMs - last_attempt_at_ms) >= retry_interval_ms) ORDER BY created_at ASC"; // Bound the sweep batch (oldest-first). 0 = unlimited (legacy). if (limit > 0) { cmd.CommandText += "\n LIMIT @limit"; cmd.Parameters.AddWithValue("@limit", limit); } cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending); cmd.Parameters.AddWithValue("@nowMs", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); return await ReadMessagesAsync(cmd); } /// /// Updates a message after a delivery attempt. /// /// The message with updated retry count, status, and last error. /// A task that represents the asynchronous operation. public async Task UpdateMessageAsync(StoreAndForwardMessage message) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" UPDATE sf_messages SET retry_count = @retryCount, last_attempt_at = @lastAttempt, last_attempt_at_ms = @lastAttemptMs, status = @status, last_error = @lastError WHERE id = @id"; cmd.Parameters.AddWithValue("@id", message.Id); cmd.Parameters.AddWithValue("@retryCount", message.RetryCount); cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue ? message.LastAttemptAt.Value.ToString("O") : DBNull.Value); cmd.Parameters.AddWithValue("@lastAttemptMs", (object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value); cmd.Parameters.AddWithValue("@status", (int)message.Status); cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value); await cmd.ExecuteNonQueryAsync(); } /// /// Updates a message after a delivery attempt, but only if the row is still /// in the expected status. Returns true if the row was updated, false if it had /// already been changed (e.g. an operator retried or discarded the message) and so /// was skipped. /// /// The message with the updated values to persist. /// The status the row must currently have for the update to proceed. /// A task that resolves to true if the row was updated; false if its status had already changed. public async Task UpdateMessageIfStatusAsync( StoreAndForwardMessage message, StoreAndForwardMessageStatus expectedStatus) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" UPDATE sf_messages SET retry_count = @retryCount, last_attempt_at = @lastAttempt, last_attempt_at_ms = @lastAttemptMs, status = @status, last_error = @lastError WHERE id = @id AND status = @expectedStatus"; cmd.Parameters.AddWithValue("@id", message.Id); cmd.Parameters.AddWithValue("@retryCount", message.RetryCount); cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue ? message.LastAttemptAt.Value.ToString("O") : DBNull.Value); cmd.Parameters.AddWithValue("@lastAttemptMs", (object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value); cmd.Parameters.AddWithValue("@status", (int)message.Status); cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value); cmd.Parameters.AddWithValue("@expectedStatus", (int)expectedStatus); var rows = await cmd.ExecuteNonQueryAsync(); return rows > 0; } /// /// Removes a successfully delivered message. /// /// The id of the message to remove. /// A task that represents the asynchronous operation. public async Task RemoveMessageAsync(string messageId) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id"; cmd.Parameters.AddWithValue("@id", messageId); await cmd.ExecuteNonQueryAsync(); } /// /// Gets all parked messages, optionally filtered by category, with pagination. /// /// Optional category filter; null returns parked messages from all categories. /// 1-based page number. /// Maximum number of messages to return per page. /// A task that resolves to the page of parked messages and the total count of matching rows. public async Task<(List Messages, int TotalCount)> GetParkedMessagesAsync( StoreAndForwardCategory? category = null, int pageNumber = 1, int pageSize = 50) { await using var connection = OpenConnection(); await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(); // Count await using var countCmd = connection.CreateCommand(); countCmd.Transaction = transaction; countCmd.CommandText = category.HasValue ? "SELECT COUNT(*) FROM sf_messages WHERE status = @parked AND category = @category" : "SELECT COUNT(*) FROM sf_messages WHERE status = @parked"; countCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); if (category.HasValue) countCmd.Parameters.AddWithValue("@category", (int)category.Value); var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync()); // Page await using var pageCmd = connection.CreateCommand(); pageCmd.Transaction = transaction; var categoryFilter = category.HasValue ? " AND category = @category" : ""; pageCmd.CommandText = $@" SELECT id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance, execution_id, source_script, parent_execution_id FROM sf_messages WHERE status = @parked{categoryFilter} ORDER BY created_at ASC LIMIT @limit OFFSET @offset"; pageCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); if (category.HasValue) pageCmd.Parameters.AddWithValue("@category", (int)category.Value); pageCmd.Parameters.AddWithValue("@limit", pageSize); pageCmd.Parameters.AddWithValue("@offset", (pageNumber - 1) * pageSize); var messages = await ReadMessagesAsync(pageCmd); await transaction.CommitAsync(); return (messages, totalCount); } /// /// Moves a parked message back to pending for retry. /// /// The id of the parked message to move back to Pending. /// A task that resolves to true if the message was found and reset to Pending; false if not found or not in Parked status. public async Task RetryParkedMessageAsync(string messageId) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" UPDATE sf_messages SET status = @pending, retry_count = 0, last_error = NULL, last_attempt_at = NULL, last_attempt_at_ms = NULL WHERE id = @id AND status = @parked"; cmd.Parameters.AddWithValue("@id", messageId); cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending); cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); var rows = await cmd.ExecuteNonQueryAsync(); return rows > 0; } /// /// Permanently discards a parked message. /// /// The id of the parked message to discard. /// A task that resolves to true if the message was found and deleted; false if not found or not in Parked status. public async Task DiscardParkedMessageAsync(string messageId) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id AND status = @parked"; cmd.Parameters.AddWithValue("@id", messageId); cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); var rows = await cmd.ExecuteNonQueryAsync(); return rows > 0; } /// /// Gets buffer depth by category (count of pending messages per category). /// /// A task that resolves to a dictionary mapping each category to its pending message count. public async Task> GetBufferDepthByCategoryAsync() { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" SELECT category, COUNT(*) as cnt FROM sf_messages WHERE status = @pending GROUP BY category"; cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending); var result = new Dictionary(); await using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { var category = (StoreAndForwardCategory)reader.GetInt32(0); var count = reader.GetInt32(1); result[category] = count; } return result; } /// /// Verifies messages are NOT deleted when an instance is deleted. /// Returns the count of messages for a given origin instance. /// /// The origin instance name to count messages for. /// A task that resolves to the number of messages whose origin instance matches . public async Task GetMessageCountByOriginInstanceAsync(string instanceName) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" SELECT COUNT(*) FROM sf_messages WHERE origin_instance = @origin"; cmd.Parameters.AddWithValue("@origin", instanceName); return Convert.ToInt32(await cmd.ExecuteScalarAsync()); } /// /// Gets a message by ID. /// /// The id of the message to retrieve. /// A task that resolves to the matching message, or null if not found. public async Task GetMessageByIdAsync(string messageId) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = @" SELECT id, category, target, payload_json, retry_count, max_retries, retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance, execution_id, source_script, parent_execution_id FROM sf_messages WHERE id = @id"; cmd.Parameters.AddWithValue("@id", messageId); var messages = await ReadMessagesAsync(cmd); return messages.FirstOrDefault(); } /// /// Gets the count of parked messages (for health reporting). /// /// A task that resolves to the number of messages currently in Parked status. public async Task GetParkedMessageCountAsync() { await using var conn = OpenConnection(); await using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM sf_messages WHERE status = @parked"; cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); var result = await cmd.ExecuteScalarAsync(); return Convert.ToInt32(result); } /// /// Gets the created_at of the oldest parked message, or null when /// no rows are parked. Backs the parked-retention aging signal on the site health /// report: parked rows persist until an operator acts, so their age — not /// just their count — is what tells an operator a forgotten site's backlog is /// growing stale. /// /// A task that resolves to the oldest parked row's creation time, or null if none are parked. public async Task GetOldestParkedCreatedAtAsync() { await using var conn = OpenConnection(); await using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT MIN(created_at) FROM sf_messages WHERE status = @parked"; cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked); var result = await cmd.ExecuteScalarAsync(); return result is null or DBNull ? null : DateTimeOffset.Parse((string)result); } /// /// Gets total message count by status. /// /// The status to filter by. /// A task that resolves to the count of messages with the specified status. public async Task GetMessageCountByStatusAsync(StoreAndForwardMessageStatus status) { await using var connection = OpenConnection(); await using var cmd = connection.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM sf_messages WHERE status = @status"; cmd.Parameters.AddWithValue("@status", (int)status); return Convert.ToInt32(await cmd.ExecuteScalarAsync()); } private static async Task> ReadMessagesAsync(SqliteCommand cmd) { var results = new List(); await using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) { results.Add(new StoreAndForwardMessage { Id = reader.GetString(0), Category = (StoreAndForwardCategory)reader.GetInt32(1), Target = reader.GetString(2), PayloadJson = reader.GetString(3), RetryCount = reader.GetInt32(4), MaxRetries = reader.GetInt32(5), RetryIntervalMs = reader.GetInt64(6), CreatedAt = DateTimeOffset.Parse(reader.GetString(7)), LastAttemptAt = reader.IsDBNull(8) ? null : DateTimeOffset.Parse(reader.GetString(8)), Status = (StoreAndForwardMessageStatus)reader.GetInt32(9), LastError = reader.IsDBNull(10) ? null : reader.GetString(10), OriginInstanceName = reader.IsDBNull(11) ? null : reader.GetString(11), // Rows persisted before the // additive migration have no execution_id / source_script value; // IsDBNull guards keep those reading back as null (back-compat). // Guid.TryParse (not Parse) guards the retry sweep: a corrupt // non-null execution_id is treated as "no execution id" rather // than throwing FormatException and aborting the whole sweep. ExecutionId = ParseGuidColumn(reader, 12), SourceScript = reader.IsDBNull(13) ? null : reader.GetString(13), // Rows persisted // before the additive migration have no parent_execution_id // value; the IsDBNull guard inside ParseGuidColumn keeps those // reading back as null (back-compat). Guid.TryParse (not Parse) // guards the retry sweep against a corrupt non-null value. ParentExecutionId = ParseGuidColumn(reader, 14) }); } return results; } /// /// Defensively reads a nullable GUID column (execution_id or /// parent_execution_id). A null value (legacy pre-migration /// rows) and a malformed non-null value both yield null — a corrupt /// id must not throw and abort the retry sweep, which reads many rows. /// private static Guid? ParseGuidColumn(System.Data.Common.DbDataReader reader, int ordinal) { if (reader.IsDBNull(ordinal)) { return null; } return Guid.TryParse(reader.GetString(ordinal), out var value) ? value : null; } }