using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.LocalDb;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.StoreAndForward;
///
/// SQLite persistence layer for store-and-forward messages.
/// Uses direct Microsoft.Data.Sqlite (not EF Core) for lightweight site-side storage.
/// No max buffer size per design decision.
///
public class StoreAndForwardStorage
{
private readonly ILocalDb _localDb;
private readonly ILogger _logger;
///
/// Initializes the store over the consolidated site database and applies the schema.
///
///
/// The consolidated site database. Every connection it hands out is already open and carries
/// the per-connection pragmas (including busy_timeout) plus the zb_hlc_next()
/// UDF that sf_messages' capture triggers call — which is exactly why the store no
/// longer opens its own from a connection string. A raw
/// connection would lack the UDF and every write to the replicated table would fail closed.
///
/// Logger for diagnostics.
public StoreAndForwardStorage(ILocalDb localDb, ILogger logger)
{
ArgumentNullException.ThrowIfNull(localDb);
ArgumentNullException.ThrowIfNull(logger);
_localDb = localDb;
_logger = logger;
}
///
/// Creates the sf_messages table if it does not exist.
///
/// A task that represents the asynchronous operation.
public Task InitializeAsync()
{
// No directory creation and no journal-mode pragma here any more: LocalDb owns
// the file (it creates the directory) and sets WAL plus the per-connection
// pragmas on every connection it hands out.
using var connection = OpenConnection();
// The DDL itself lives in StoreAndForwardSchema so the Host can apply it to a
// LocalDb-managed connection before RegisterReplicated installs the capture
// triggers. The store still calls it, so a directly-constructed store (tests,
// tooling) remains self-sufficient.
StoreAndForwardSchema.Apply(connection);
_logger.LogInformation("Store-and-forward SQLite storage initialized");
return Task.CompletedTask;
}
///
/// CreateConnection returns an ALREADY-OPEN, pragma-configured connection with the
/// zb_hlc_next() UDF registered. Calling OpenAsync on it throws, and a raw
/// would lack the UDF, making every capture trigger fail
/// closed.
///
private SqliteConnection OpenConnection() => _localDb.CreateConnection();
///
/// INSERT statement for a full message row. Shared by
/// and ; bind with
/// so the column list and the parameters never drift apart.
///
private const string InsertMessageSql = @"
INSERT INTO sf_messages (id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, last_attempt_at_ms, status, last_error,
origin_instance, execution_id, source_script, parent_execution_id)
VALUES (@id, @category, @target, @payload, @retryCount, @maxRetries,
@retryIntervalMs, @createdAt, @lastAttempt, @lastAttemptMs, @status, @lastError,
@origin, @executionId, @sourceScript, @parentExecutionId)";
///
/// Binds every column parameter for a full message row (used by
/// ). GUID ids are stored in canonical "D" string
/// form and the epoch-ms sibling of last_attempt_at is derived here so a single
/// place owns the mapping.
///
private static void BindMessageParameters(SqliteCommand cmd, StoreAndForwardMessage message)
{
cmd.Parameters.AddWithValue("@id", message.Id);
cmd.Parameters.AddWithValue("@category", (int)message.Category);
cmd.Parameters.AddWithValue("@target", message.Target);
cmd.Parameters.AddWithValue("@payload", message.PayloadJson);
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
cmd.Parameters.AddWithValue("@maxRetries", message.MaxRetries);
cmd.Parameters.AddWithValue("@retryIntervalMs", message.RetryIntervalMs);
cmd.Parameters.AddWithValue("@createdAt", message.CreatedAt.ToString("O"));
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
cmd.Parameters.AddWithValue("@lastAttemptMs",
(object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value);
cmd.Parameters.AddWithValue("@status", (int)message.Status);
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
cmd.Parameters.AddWithValue("@origin", (object?)message.OriginInstanceName ?? DBNull.Value);
// GUID ids stored as canonical "D" strings; null when not threaded.
cmd.Parameters.AddWithValue("@executionId",
message.ExecutionId.HasValue ? message.ExecutionId.Value.ToString("D") : DBNull.Value);
cmd.Parameters.AddWithValue("@sourceScript", (object?)message.SourceScript ?? DBNull.Value);
cmd.Parameters.AddWithValue("@parentExecutionId",
message.ParentExecutionId.HasValue ? message.ParentExecutionId.Value.ToString("D") : DBNull.Value);
}
///
/// Enqueues a new message with Pending status.
///
/// The message to enqueue.
/// A task that represents the asynchronous operation.
public async Task EnqueueAsync(StoreAndForwardMessage message)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = InsertMessageSql;
BindMessageParameters(cmd, message);
await cmd.ExecuteNonQueryAsync();
}
///
/// Returns every buffered message regardless of status, oldest-first, up to
/// rows, plus a flag indicating the buffer holds more
/// than that. Anti-entropy groundwork: the active node snapshots its buffer for
/// a standby resync. One extra row is fetched (limit + 1) purely to
/// compute the truncation flag without a second COUNT round-trip.
///
/// Maximum rows to return; must be positive.
/// The oldest-first page (at most rows) and whether more rows exist beyond it.
public async Task<(List Messages, bool Truncated)> GetAllMessagesAsync(int limit)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
SELECT id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance,
execution_id, source_script, parent_execution_id
FROM sf_messages
ORDER BY created_at ASC
LIMIT @limitPlusOne";
cmd.Parameters.AddWithValue("@limitPlusOne", limit + 1);
var rows = await ReadMessagesAsync(cmd);
var truncated = rows.Count > limit;
return (rows.Take(limit).ToList(), truncated);
}
///
/// Inserts a message, or updates every mutable column in place if a row with the
/// same id already exists (ON CONFLICT(id) DO UPDATE). Used by the standby
/// node when applying a replicated Add/Park/Requeue: a Park/Requeue whose original
/// Add was lost self-heals (the full message rides in the operation), and a
/// duplicate Add (e.g. after an anti-entropy resync) applies newest-wins instead of
/// violating the primary key. The original created_at is preserved — it
/// orders the retry sweep and must never be overwritten on update.
///
/// The message to insert or update.
/// A task that represents the asynchronous operation.
public async Task UpsertMessageAsync(StoreAndForwardMessage message)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
INSERT INTO sf_messages (id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, last_attempt_at_ms, status, last_error,
origin_instance, execution_id, source_script, parent_execution_id)
VALUES (@id, @category, @target, @payload, @retryCount, @maxRetries,
@retryIntervalMs, @createdAt, @lastAttempt, @lastAttemptMs, @status, @lastError,
@origin, @executionId, @sourceScript, @parentExecutionId)
ON CONFLICT(id) DO UPDATE SET
category = excluded.category,
target = excluded.target,
payload_json = excluded.payload_json,
retry_count = excluded.retry_count,
max_retries = excluded.max_retries,
retry_interval_ms = excluded.retry_interval_ms,
last_attempt_at = excluded.last_attempt_at,
last_attempt_at_ms = excluded.last_attempt_at_ms,
status = excluded.status,
last_error = excluded.last_error,
origin_instance = excluded.origin_instance,
execution_id = excluded.execution_id,
source_script = excluded.source_script,
parent_execution_id = excluded.parent_execution_id";
cmd.Parameters.AddWithValue("@id", message.Id);
cmd.Parameters.AddWithValue("@category", (int)message.Category);
cmd.Parameters.AddWithValue("@target", message.Target);
cmd.Parameters.AddWithValue("@payload", message.PayloadJson);
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
cmd.Parameters.AddWithValue("@maxRetries", message.MaxRetries);
cmd.Parameters.AddWithValue("@retryIntervalMs", message.RetryIntervalMs);
cmd.Parameters.AddWithValue("@createdAt", message.CreatedAt.ToString("O"));
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
cmd.Parameters.AddWithValue("@lastAttemptMs",
(object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value);
cmd.Parameters.AddWithValue("@status", (int)message.Status);
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
cmd.Parameters.AddWithValue("@origin", (object?)message.OriginInstanceName ?? DBNull.Value);
cmd.Parameters.AddWithValue("@executionId",
message.ExecutionId.HasValue ? message.ExecutionId.Value.ToString("D") : DBNull.Value);
cmd.Parameters.AddWithValue("@sourceScript", (object?)message.SourceScript ?? DBNull.Value);
cmd.Parameters.AddWithValue("@parentExecutionId",
message.ParentExecutionId.HasValue ? message.ParentExecutionId.Value.ToString("D") : DBNull.Value);
await cmd.ExecuteNonQueryAsync();
}
///
/// Gets all messages that are due for retry (Pending status, last attempt older than retry interval).
///
/// Maximum number of messages to return, or 0 for no limit.
/// A task that resolves to the list of messages due for retry, ordered by creation time ascending.
public async Task> GetMessagesForRetryAsync(int limit = 0)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
SELECT id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance,
execution_id, source_script, parent_execution_id
FROM sf_messages
WHERE status = @pending
AND (last_attempt_at_ms IS NULL
OR retry_interval_ms = 0
OR (@nowMs - last_attempt_at_ms) >= retry_interval_ms)
ORDER BY created_at ASC";
// Bound the sweep batch (oldest-first). 0 = unlimited (legacy).
if (limit > 0)
{
cmd.CommandText += "\n LIMIT @limit";
cmd.Parameters.AddWithValue("@limit", limit);
}
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
cmd.Parameters.AddWithValue("@nowMs", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds());
return await ReadMessagesAsync(cmd);
}
///
/// Updates a message after a delivery attempt.
///
/// The message with updated retry count, status, and last error.
/// A task that represents the asynchronous operation.
public async Task UpdateMessageAsync(StoreAndForwardMessage message)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
UPDATE sf_messages
SET retry_count = @retryCount,
last_attempt_at = @lastAttempt,
last_attempt_at_ms = @lastAttemptMs,
status = @status,
last_error = @lastError
WHERE id = @id";
cmd.Parameters.AddWithValue("@id", message.Id);
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
cmd.Parameters.AddWithValue("@lastAttemptMs",
(object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value);
cmd.Parameters.AddWithValue("@status", (int)message.Status);
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
await cmd.ExecuteNonQueryAsync();
}
///
/// Updates a message after a delivery attempt, but only if the row is still
/// in the expected status. Returns true if the row was updated, false if it had
/// already been changed (e.g. an operator retried or discarded the message) and so
/// was skipped.
///
/// The message with the updated values to persist.
/// The status the row must currently have for the update to proceed.
/// A task that resolves to true if the row was updated; false if its status had already changed.
public async Task UpdateMessageIfStatusAsync(
StoreAndForwardMessage message,
StoreAndForwardMessageStatus expectedStatus)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
UPDATE sf_messages
SET retry_count = @retryCount,
last_attempt_at = @lastAttempt,
last_attempt_at_ms = @lastAttemptMs,
status = @status,
last_error = @lastError
WHERE id = @id AND status = @expectedStatus";
cmd.Parameters.AddWithValue("@id", message.Id);
cmd.Parameters.AddWithValue("@retryCount", message.RetryCount);
cmd.Parameters.AddWithValue("@lastAttempt", message.LastAttemptAt.HasValue
? message.LastAttemptAt.Value.ToString("O") : DBNull.Value);
cmd.Parameters.AddWithValue("@lastAttemptMs",
(object?)message.LastAttemptAt?.ToUnixTimeMilliseconds() ?? DBNull.Value);
cmd.Parameters.AddWithValue("@status", (int)message.Status);
cmd.Parameters.AddWithValue("@lastError", (object?)message.LastError ?? DBNull.Value);
cmd.Parameters.AddWithValue("@expectedStatus", (int)expectedStatus);
var rows = await cmd.ExecuteNonQueryAsync();
return rows > 0;
}
///
/// Removes a successfully delivered message.
///
/// The id of the message to remove.
/// A task that represents the asynchronous operation.
public async Task RemoveMessageAsync(string messageId)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id";
cmd.Parameters.AddWithValue("@id", messageId);
await cmd.ExecuteNonQueryAsync();
}
///
/// Gets all parked messages, optionally filtered by category, with pagination.
///
/// Optional category filter; null returns parked messages from all categories.
/// 1-based page number.
/// Maximum number of messages to return per page.
/// A task that resolves to the page of parked messages and the total count of matching rows.
public async Task<(List Messages, int TotalCount)> GetParkedMessagesAsync(
StoreAndForwardCategory? category = null,
int pageNumber = 1,
int pageSize = 50)
{
await using var connection = OpenConnection();
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync();
// Count
await using var countCmd = connection.CreateCommand();
countCmd.Transaction = transaction;
countCmd.CommandText = category.HasValue
? "SELECT COUNT(*) FROM sf_messages WHERE status = @parked AND category = @category"
: "SELECT COUNT(*) FROM sf_messages WHERE status = @parked";
countCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
if (category.HasValue) countCmd.Parameters.AddWithValue("@category", (int)category.Value);
var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync());
// Page
await using var pageCmd = connection.CreateCommand();
pageCmd.Transaction = transaction;
var categoryFilter = category.HasValue ? " AND category = @category" : "";
pageCmd.CommandText = $@"
SELECT id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance,
execution_id, source_script, parent_execution_id
FROM sf_messages
WHERE status = @parked{categoryFilter}
ORDER BY created_at ASC
LIMIT @limit OFFSET @offset";
pageCmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
if (category.HasValue) pageCmd.Parameters.AddWithValue("@category", (int)category.Value);
pageCmd.Parameters.AddWithValue("@limit", pageSize);
pageCmd.Parameters.AddWithValue("@offset", (pageNumber - 1) * pageSize);
var messages = await ReadMessagesAsync(pageCmd);
await transaction.CommitAsync();
return (messages, totalCount);
}
///
/// Moves a parked message back to pending for retry.
///
/// The id of the parked message to move back to Pending.
/// A task that resolves to true if the message was found and reset to Pending; false if not found or not in Parked status.
public async Task RetryParkedMessageAsync(string messageId)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
UPDATE sf_messages
SET status = @pending, retry_count = 0, last_error = NULL,
last_attempt_at = NULL, last_attempt_at_ms = NULL
WHERE id = @id AND status = @parked";
cmd.Parameters.AddWithValue("@id", messageId);
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
var rows = await cmd.ExecuteNonQueryAsync();
return rows > 0;
}
///
/// Permanently discards a parked message.
///
/// The id of the parked message to discard.
/// A task that resolves to true if the message was found and deleted; false if not found or not in Parked status.
public async Task DiscardParkedMessageAsync(string messageId)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = "DELETE FROM sf_messages WHERE id = @id AND status = @parked";
cmd.Parameters.AddWithValue("@id", messageId);
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
var rows = await cmd.ExecuteNonQueryAsync();
return rows > 0;
}
///
/// Gets buffer depth by category (count of pending messages per category).
///
/// A task that resolves to a dictionary mapping each category to its pending message count.
public async Task> GetBufferDepthByCategoryAsync()
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
SELECT category, COUNT(*) as cnt
FROM sf_messages
WHERE status = @pending
GROUP BY category";
cmd.Parameters.AddWithValue("@pending", (int)StoreAndForwardMessageStatus.Pending);
var result = new Dictionary();
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
var category = (StoreAndForwardCategory)reader.GetInt32(0);
var count = reader.GetInt32(1);
result[category] = count;
}
return result;
}
///
/// Verifies messages are NOT deleted when an instance is deleted.
/// Returns the count of messages for a given origin instance.
///
/// The origin instance name to count messages for.
/// A task that resolves to the number of messages whose origin instance matches .
public async Task GetMessageCountByOriginInstanceAsync(string instanceName)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
SELECT COUNT(*)
FROM sf_messages
WHERE origin_instance = @origin";
cmd.Parameters.AddWithValue("@origin", instanceName);
return Convert.ToInt32(await cmd.ExecuteScalarAsync());
}
///
/// Gets a message by ID.
///
/// The id of the message to retrieve.
/// A task that resolves to the matching message, or null if not found.
public async Task GetMessageByIdAsync(string messageId)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = @"
SELECT id, category, target, payload_json, retry_count, max_retries,
retry_interval_ms, created_at, last_attempt_at, status, last_error, origin_instance,
execution_id, source_script, parent_execution_id
FROM sf_messages
WHERE id = @id";
cmd.Parameters.AddWithValue("@id", messageId);
var messages = await ReadMessagesAsync(cmd);
return messages.FirstOrDefault();
}
///
/// Gets the count of parked messages (for health reporting).
///
/// A task that resolves to the number of messages currently in Parked status.
public async Task GetParkedMessageCountAsync()
{
await using var conn = OpenConnection();
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM sf_messages WHERE status = @parked";
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
var result = await cmd.ExecuteScalarAsync();
return Convert.ToInt32(result);
}
///
/// Gets the created_at of the oldest parked message, or null when
/// no rows are parked. Backs the parked-retention aging signal on the site health
/// report: parked rows persist until an operator acts, so their age — not
/// just their count — is what tells an operator a forgotten site's backlog is
/// growing stale.
///
/// A task that resolves to the oldest parked row's creation time, or null if none are parked.
public async Task GetOldestParkedCreatedAtAsync()
{
await using var conn = OpenConnection();
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT MIN(created_at) FROM sf_messages WHERE status = @parked";
cmd.Parameters.AddWithValue("@parked", (int)StoreAndForwardMessageStatus.Parked);
var result = await cmd.ExecuteScalarAsync();
return result is null or DBNull
? null
: DateTimeOffset.Parse((string)result);
}
///
/// Gets total message count by status.
///
/// The status to filter by.
/// A task that resolves to the count of messages with the specified status.
public async Task GetMessageCountByStatusAsync(StoreAndForwardMessageStatus status)
{
await using var connection = OpenConnection();
await using var cmd = connection.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM sf_messages WHERE status = @status";
cmd.Parameters.AddWithValue("@status", (int)status);
return Convert.ToInt32(await cmd.ExecuteScalarAsync());
}
private static async Task> ReadMessagesAsync(SqliteCommand cmd)
{
var results = new List();
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
results.Add(new StoreAndForwardMessage
{
Id = reader.GetString(0),
Category = (StoreAndForwardCategory)reader.GetInt32(1),
Target = reader.GetString(2),
PayloadJson = reader.GetString(3),
RetryCount = reader.GetInt32(4),
MaxRetries = reader.GetInt32(5),
RetryIntervalMs = reader.GetInt64(6),
CreatedAt = DateTimeOffset.Parse(reader.GetString(7)),
LastAttemptAt = reader.IsDBNull(8) ? null : DateTimeOffset.Parse(reader.GetString(8)),
Status = (StoreAndForwardMessageStatus)reader.GetInt32(9),
LastError = reader.IsDBNull(10) ? null : reader.GetString(10),
OriginInstanceName = reader.IsDBNull(11) ? null : reader.GetString(11),
// Rows persisted before the
// additive migration have no execution_id / source_script value;
// IsDBNull guards keep those reading back as null (back-compat).
// Guid.TryParse (not Parse) guards the retry sweep: a corrupt
// non-null execution_id is treated as "no execution id" rather
// than throwing FormatException and aborting the whole sweep.
ExecutionId = ParseGuidColumn(reader, 12),
SourceScript = reader.IsDBNull(13) ? null : reader.GetString(13),
// Rows persisted
// before the additive migration have no parent_execution_id
// value; the IsDBNull guard inside ParseGuidColumn keeps those
// reading back as null (back-compat). Guid.TryParse (not Parse)
// guards the retry sweep against a corrupt non-null value.
ParentExecutionId = ParseGuidColumn(reader, 14)
});
}
return results;
}
///
/// Defensively reads a nullable GUID column (execution_id or
/// parent_execution_id). A null value (legacy pre-migration
/// rows) and a malformed non-null value both yield null — a corrupt
/// id must not throw and abort the retry sweep, which reads many rows.
///
private static Guid? ParseGuidColumn(System.Data.Common.DbDataReader reader, int ordinal)
{
if (reader.IsDBNull(ordinal))
{
return null;
}
return Guid.TryParse(reader.GetString(ordinal), out var value)
? value
: null;
}
}