dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

merge(agent-b): rows 81+84 — storage cap now measures site_events' own footprint (sampled page-layout model; dbstat probed unavailable and pinned); backlog note once per tick

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-17 03:59:21 -04:00
parent 817a72108e ddb3c32e06
commit 5162408a9f
10 changed files with 1171 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteEventLogging/EventLogPurgeService.cs
+294 -42
View File
@@ -1,3 +1,5 @@
using System.Globalization;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
@@ -60,6 +62,40 @@ public class EventLogPurgeService : BackgroundService
/// <summary>Number of events deleted per retention-purge batch — same size as <see cref="CapPurgeBatchSize"/>.</summary>
private const int RetentionPurgeBatchSize = 1000;
/// <summary>
/// Rows sampled from each end of <c>site_events</c> when sizing an average row. See
/// <see cref="EstimateSiteEventsSize"/>.
/// </summary>
private const int SizeSampleRows = 500;
/// <summary>
/// Per-row bookkeeping a table B-tree leaf spends on top of the row's own bytes: the
/// payload-length varint, the rowid varint and the 2-byte cell-pointer slot.
/// </summary>
private const int TableCellOverheadBytes = 16;
/// <summary>
/// Bytes at the end of a page SQLite will not fill with local payload — the
/// <c>pageSize - 35</c> maximum-local-payload rule for table leaves.
/// </summary>
private const int PageReservedBytes = 35;
/// <summary>
/// Average proportion of a table leaf page that actually holds rows. B-tree splits leave
/// pages part-empty; 0.78 is the measured mean across the workloads in
/// <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c> §3.
/// </summary>
private const double TablePageFillFactor = 0.78;
/// <summary>Number of indexes on <c>site_events</c> (timestamp, event_type, instance_id, severity).</summary>
private const int IndexedColumnCount = 4;
/// <summary>Per-entry bookkeeping in an index B-tree leaf: key-length varint, rowid and cell pointer.</summary>
private const int IndexEntryOverheadBytes = 8;
/// <summary>Average fill of an index leaf page; measured alongside <see cref="TablePageFillFactor"/>.</summary>
private const double IndexPageFillFactor = 0.75;
/// <summary>Minimum spacing between ceiling-reached log lines; see <see cref="LogCeilingReached"/>.</summary>
private static readonly TimeSpan CeilingLogInterval = TimeSpan.FromMinutes(15);
@@ -186,6 +222,14 @@ public class EventLogPurgeService : BackgroundService
PurgeByRetention(budget);
PurgeByStorageCap(budget);
// Register row 84: emitted ONCE per tick, from the tick's shared budget, rather
// than once per phase. A tick in which both the retention purge and the cap trim
// deleted rows used to log two identical Information notes; the number an operator
// wants next to the oplog spike is the tick total anyway, which is exactly what
// the shared budget already holds.
if (budget.Deleted > 0)
LogReplicationBacklogNote(budget.Deleted);
if (!budget.StoppedAtCeiling)
return false;
@@ -290,10 +334,7 @@ public class EventLogPurgeService : BackgroundService
}
if (totalDeleted > 0)
{
_logger.LogInformation("Purged {Count} events older than {Days} days", totalDeleted, _options.RetentionDays);
LogReplicationBacklogNote(totalDeleted);
}
}
/// <summary>
@@ -318,9 +359,17 @@ public class EventLogPurgeService : BackgroundService
/// <summary>
/// Emits the operator breadcrumb that explains the oplog backlog spike a purge produces on a
/// replication-enabled node. No-op when this node has no replication peer.
/// replication-enabled node — <b>once per purge tick</b>. No-op when this node has no
/// replication peer.
/// </summary>
/// <remarks>
/// <para>
/// Called from <see cref="RunPurge"/> rather than from each phase (register row 84). Both the
/// retention purge and the cap trim generate the same CDC cost, and a tick in which both
/// deleted rows used to emit two identical Information notes. One note carrying the tick's
/// combined total is what an operator correlating an oplog spike actually wants.
/// </para>
/// <para>
/// <c>site_events</c> is a replicated table, and a retention/cap DELETE is captured by CDC
/// exactly like an ordinary write — there is deliberately no purge-exemption path (LocalDb
/// Phase 2: CDC does all three jobs). The daily purge therefore pushes one oplog row per
@@ -329,8 +378,12 @@ public class EventLogPurgeService : BackgroundService
/// on its own; without this line an operator correlating the spike has nothing in the log to
/// tie it to. Information level because it is only interesting next to the purge line it
/// follows — the spike itself is not a fault.
/// </para>
/// </remarks>
/// <param name="deletedRows">Number of rows the purge just deleted; always &gt; 0 at the call sites.</param>
/// <param name="deletedRows">
/// Rows deleted by the WHOLE tick — retention purge and cap trim combined, read off the
/// shared <see cref="PurgeBudget"/>. Always &gt; 0 at the single call site.
/// </param>
private void LogReplicationBacklogNote(int deletedRows)
{
bool replicated;
@@ -358,13 +411,52 @@ public class EventLogPurgeService : BackgroundService
_options.MaxPurgedRowsPerTick > 0 ? _options.MaxPurgedRowsPerTick.ToString() : "unlimited");
}
/// <summary>
/// Trims the oldest events until <c>site_events</c>' own footprint is back under
/// <see cref="SiteEventLogOptions.MaxStorageMb"/>.
/// </summary>
/// <remarks>
/// <para>
/// <b>The cap is table-scoped as of register row 81.</b> It used to compare
/// <see cref="SiteEventLogOptions.MaxStorageMb"/> against the whole consolidated LocalDb
/// file, which on a replication-enabled node made the trim unwinnable: every batch freed
/// event pages and added an equal number of oplog rows and row-version tombstones to the
/// same file, so the measured size did not fall and the progress guard stopped the trim
/// 2.5x over cap with 226 k rows still present. The cap now measures
/// <see cref="EstimateSiteEventsSize"/> — <c>site_events</c> and its four indexes only —
/// so replication bookkeeping can no longer consume, or block, the event log's budget.
/// </para>
/// <para>
/// Bytes-per-row is sampled ONCE per tick and held fixed for the whole loop, while the row
/// count is re-read after each batch. That split is deliberate: re-sampling row width every
/// batch would make the estimate jitter as the trim consumes the head of the table, and a
/// sample that happened to widen could stall the loop through the progress guard below —
/// the same class of self-defeating measurement row 81 was about.
/// </para>
/// </remarks>
/// <param name="budget">The tick's shared delete budget, already drawn down by the retention purge.</param>
private void PurgeByStorageCap(PurgeBudget budget)
{
var capBytes = (long)_options.MaxStorageMb * 1024 * 1024;
var currentSizeBytes = GetDatabaseSizeBytes();
var estimate = EstimateSiteEventsSize();
if (currentSizeBytes <= capBytes)
if (estimate.RowCount <= 0)
return;
// Wipe-prevention backstop, first of two — checked BEFORE the cap comparison, because a
// zero row width makes "site_events is over cap" unanswerable rather than false. An
// unusable width (empty sample, degenerate estimate) used to have no analogue: the old
// loop's answer to an unanswerable cap was to keep deleting. Refuse instead.
if (estimate.BytesPerRow <= 0)
{
_logger.LogWarning(
"Storage-cap trim skipped: site_events row width could not be sampled ({Rows} rows). " +
"No rows are deleted rather than trimming against an unusable size estimate.",
estimate.RowCount);
return;
}
if (estimate.EstimatedBytes <= capBytes)
return;
// Over cap but out of budget: flag the catch-up tick and stay silent rather than logging
@@ -378,14 +470,18 @@ public class EventLogPurgeService : BackgroundService
}
_logger.LogWarning(
"Event log size {Size:F1} MB exceeds cap {Cap} MB — purging oldest events",
currentSizeBytes / (1024.0 * 1024.0), _options.MaxStorageMb);
"site_events occupies about {Size:F1} MB across {Rows} rows — over the {Cap} MB cap. " +
"Trimming the oldest events. (The cap measures the event table and its indexes only, " +
"not the whole consolidated LocalDb file.)",
estimate.EstimatedBytes / (1024.0 * 1024.0), estimate.RowCount, _options.MaxStorageMb);
// Delete the oldest events in batches until the database is under the cap.
// The loop also stops if the on-disk size fails to decrease across an
// iteration (e.g. if vacuum cannot reclaim space), so a cap that can never
// be met does not silently empty the entire table.
// Delete the oldest events in batches until the ESTIMATED TABLE size is under the cap.
// Wipe-prevention backstop, second of two: the loop also stops if the estimate fails to
// fall across an iteration. Replication bookkeeping can no longer trip that — the metric
// excludes it — so it now means what it says: the table is not shrinking, because a batch
// deleted nothing or because concurrent recorder writes are outrunning the trim.
var totalDeleted = 0;
var currentSizeBytes = estimate.EstimatedBytes;
while (currentSizeBytes > capBytes)
{
@@ -430,57 +526,213 @@ public class EventLogPurgeService : BackgroundService
totalDeleted += deleted;
budget.Record(deleted);
currentSizeBytes = GetDatabaseSizeBytes();
// Row count re-read from the database, not decremented locally, so concurrent
// recorder writes are visible; row width stays fixed for the tick (see remarks).
currentSizeBytes = (long)(GetSiteEventsRowCount() * estimate.BytesPerRow);
if (currentSizeBytes >= previousSizeBytes)
{
// Size is not shrinking despite deletes — stop rather than wipe the
// whole table. This should not happen now that logical size is
// measured, but guards against any future regression.
_logger.LogWarning(
"Event log size did not decrease after a cap-purge batch ({Size:F1} MB); " +
"stopping to avoid emptying the log",
"Estimated site_events size did not decrease after a cap-purge batch " +
"({Size:F1} MB); stopping to avoid emptying the log. Deletes are landing, so " +
"this means new events are arriving at least as fast as the trim removes them.",
currentSizeBytes / (1024.0 * 1024.0));
break;
}
}
// Same CDC cost as the retention purge above — a cap-driven trim is usually the
// larger of the two, so it gets the same operator breadcrumb.
if (totalDeleted > 0)
{
LogReplicationBacklogNote(totalDeleted);
}
}
/// <summary>
/// Returns the logical size of the database in bytes — only pages that hold live
/// data, excluding free pages on the freelist. Measuring logical size (rather than
/// the raw file size from <c>page_count</c>) means the storage-cap loop observes
/// space being reclaimed even if free pages have not yet been returned to the OS.
/// Returns the logical size of the WHOLE consolidated database file in bytes — only pages
/// that hold live data, excluding free pages on the freelist.
/// </summary>
/// <returns>The logical database size in bytes (used pages × page size).</returns>
/// <remarks>
/// <para>
/// <b>This is no longer what the storage cap measures</b> (register row 81,
/// <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c>). Since LocalDb
/// Phase 1/2 the file holds ten tables plus <c>__localdb_oplog</c>,
/// <c>__localdb_row_version</c> and <c>__localdb_snapshot_state</c>, so a whole-file
/// number is not <c>site_events</c>' size: on a replication-enabled node each cap-trim
/// batch freed ~390 KB of event pages while adding ~1000 oplog rows and ~1000 tombstones
/// to the same file, and the trim fought itself to a standstill 2.5x over cap.
/// </para>
/// <para>
/// It survives for two narrow jobs: it is a true UPPER BOUND on <c>site_events</c>' own
/// pages, which <see cref="EstimateSiteEventsSize"/> uses to clamp an over-eager model,
/// and it remains a useful diagnostic for the file as a whole.
/// </para>
/// </remarks>
/// <returns>The logical size of the whole database file in bytes (used pages × page size).</returns>
internal long GetDatabaseSizeBytes()
{
return _eventLogger.WithConnection(connection => ReadLogicalFileBytes(connection));
}
/// <summary>Logical size of the whole file, on an already-held connection.</summary>
/// <param name="connection">An open connection to the consolidated database.</param>
/// <returns>Used pages × page size, in bytes.</returns>
private static long ReadLogicalFileBytes(SqliteConnection connection)
{
var pageCount = ReadPragmaLong(connection, "PRAGMA page_count");
var freeListCount = ReadPragmaLong(connection, "PRAGMA freelist_count");
var pageSize = ReadPragmaLong(connection, "PRAGMA page_size");
return Math.Max(0, pageCount - freeListCount) * pageSize;
}
/// <summary>Reads a single-value PRAGMA as a 64-bit integer.</summary>
/// <param name="connection">An open connection.</param>
/// <param name="pragma">The PRAGMA statement to run.</param>
/// <returns>The PRAGMA's value.</returns>
private static long ReadPragmaLong(SqliteConnection connection, string pragma)
{
using var cmd = connection.CreateCommand();
cmd.CommandText = pragma;
return Convert.ToInt64(cmd.ExecuteScalar(), CultureInfo.InvariantCulture);
}
/// <summary>
/// How many rows <c>site_events</c> currently holds, and how many bytes of the file one
/// average row occupies — the two numbers the storage cap is expressed in.
/// </summary>
/// <param name="RowCount">Live row count at the time of the estimate.</param>
/// <param name="BytesPerRow">
/// Modelled file bytes per row, table pages plus index pages. <c>0</c> when the table is
/// empty or the sample could not be sized, which the cap trim treats as "do not trim".
/// </param>
internal readonly record struct SiteEventsSizeEstimate(long RowCount, double BytesPerRow)
{
/// <summary>Modelled total bytes <c>site_events</c> occupies in the file.</summary>
public long EstimatedBytes => (long)(RowCount * BytesPerRow);
}
/// <summary>
/// Estimates how much of the consolidated file <c>site_events</c> — and only
/// <c>site_events</c> — occupies, by sampling average row width and modelling SQLite's
/// page layout.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why a model and not a measurement.</b> SQLite can report per-table page usage
/// exactly through the <c>dbstat</c> virtual table, and that was the intended fix for
/// register row 81 — but <c>dbstat</c> is a compile-time option and the
/// <c>SQLitePCLRaw.lib.e_sqlite3</c> native build this solution ships does NOT enable it.
/// Probed directly (<c>SqliteCapabilityTests</c>): <c>PRAGMA compile_options</c> on
/// SQLite 3.53.3 lists no <c>ENABLE_DBSTAT_VTAB</c>, <c>pragma_module_list</c> returns ten
/// modules (fts3/4/5, rtree, geopoly) with neither <c>dbstat</c> nor <c>sqlite_dbpage</c>
/// among them, and selecting from <c>dbstat</c> fails with "no such table". So an exact
/// per-table page count is not obtainable from this provider at all, and the cap is
/// modelled instead. That trade is recorded in
/// <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c>.
/// </para>
/// <para>
/// <b>The model.</b> Rows are sampled from both ends of the table (<see cref="SizeSampleRows"/>
/// from each — the head is what the oldest-first trim will actually delete, the tail catches a
/// workload whose event shape has drifted) to get an average payload width and an average
/// width across the four indexed columns. Table cost is then computed as whole pages —
/// <c>ceil(rows / rowsPerPage) × pageSize</c> — rather than as a flat bytes-per-row rate,
/// because that is the term that dominates once rows approach half a page: rows too wide to
/// pack two-per-page waste the remainder, which a linear model underestimates by ~40 %.
/// Index cost is added per row from the sampled indexed-column widths, so an event stream
/// that leaves <c>instance_id</c> null is charged less than one that fills it.
/// </para>
/// <para>
/// <b>Index pages count against the cap</b> — deliberately. The four indexes are storage the
/// event log causes and owns (measured at ~34 % of the table's total footprint), and the
/// 1 GB default was sized against the 393 bytes/row figure that already included them.
/// Excluding them would silently raise real disk use by about a third at the same setting.
/// </para>
/// <para>
/// <b>Clamped by the file.</b> The whole-file logical size is a true upper bound on any one
/// table's pages, so the model is capped by it. That can only ever correct an
/// over-estimate downward; it cannot reintroduce the row-81 defect, because on a replicated
/// node the file is larger than <c>site_events</c>, not smaller, and so never binds.
/// </para>
/// <para>
/// Marked <c>virtual</c> so tests can substitute a degenerate estimate and exercise the
/// trim's backstops.
/// </para>
/// </remarks>
/// <returns>The current row count and modelled bytes per row.</returns>
internal virtual SiteEventsSizeEstimate EstimateSiteEventsSize()
{
return _eventLogger.WithConnection(connection =>
{
using var pageCountCmd = connection.CreateCommand();
pageCountCmd.CommandText = "PRAGMA page_count";
var pageCount = (long)pageCountCmd.ExecuteScalar()!;
var rowCount = ReadPragmaLong(connection, "SELECT COUNT(*) FROM site_events");
if (rowCount <= 0)
return new SiteEventsSizeEstimate(0, 0);
using var freeListCmd = connection.CreateCommand();
freeListCmd.CommandText = "PRAGMA freelist_count";
var freeListCount = (long)freeListCmd.ExecuteScalar()!;
double avgPayloadBytes;
double avgIndexedBytes;
using (var sampleCmd = connection.CreateCommand())
{
// octet_length (SQLite 3.43+) rather than length(), which counts CHARACTERS:
// a message with multi-byte UTF-8 would otherwise be under-sized. Pinned by
// SqliteCapabilityTests so a provider downgrade fails a test rather than
// silently shrinking every estimate.
sampleCmd.CommandText = $"""
SELECT AVG(payload_bytes), AVG(indexed_bytes) FROM (
SELECT payload_bytes, indexed_bytes FROM (
SELECT
octet_length(id) + octet_length(timestamp) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
+ octet_length(source) + octet_length(message)
+ COALESCE(octet_length(details), 0) AS payload_bytes,
octet_length(timestamp) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
AS indexed_bytes
FROM site_events LIMIT {SizeSampleRows}
)
UNION ALL
SELECT payload_bytes, indexed_bytes FROM (
SELECT
octet_length(id) + octet_length(timestamp) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
+ octet_length(source) + octet_length(message)
+ COALESCE(octet_length(details), 0) AS payload_bytes,
octet_length(timestamp) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
AS indexed_bytes
FROM site_events ORDER BY rowid DESC LIMIT {SizeSampleRows}
)
)
""";
using var reader = sampleCmd.ExecuteReader();
if (!reader.Read() || reader.IsDBNull(0))
return new SiteEventsSizeEstimate(rowCount, 0);
using var pageSizeCmd = connection.CreateCommand();
pageSizeCmd.CommandText = "PRAGMA page_size";
var pageSize = (long)pageSizeCmd.ExecuteScalar()!;
avgPayloadBytes = reader.GetDouble(0);
avgIndexedBytes = reader.IsDBNull(1) ? 0 : reader.GetDouble(1);
}
var usedPages = Math.Max(0, pageCount - freeListCount);
return usedPages * pageSize;
var pageSize = ReadPragmaLong(connection, "PRAGMA page_size");
var usablePerPage = (pageSize - PageReservedBytes) * TablePageFillFactor;
var avgCellBytes = avgPayloadBytes + TableCellOverheadBytes;
// At least one row per page: a row wider than a page spills to overflow pages,
// which this floor charges as a whole page each — close enough, and never under.
var rowsPerPage = Math.Max(1, (long)(usablePerPage / Math.Max(1, avgCellBytes)));
var tableBytes = (long)Math.Ceiling(rowCount / (double)rowsPerPage) * pageSize;
var indexBytesPerRow =
(avgIndexedBytes + (IndexEntryOverheadBytes * IndexedColumnCount)) / IndexPageFillFactor;
var modelled = tableBytes + (long)(rowCount * indexBytesPerRow);
var clamped = Math.Min(modelled, ReadLogicalFileBytes(connection));
return new SiteEventsSizeEstimate(rowCount, clamped / (double)rowCount);
});
}
/// <summary>Live <c>site_events</c> row count.</summary>
/// <remarks>
/// Re-read after every cap-trim batch so the trim's progress check sees concurrent
/// recorder writes, not just its own deletes. <c>virtual</c> for the backstop tests.
/// </remarks>
/// <returns>Number of rows currently in <c>site_events</c>.</returns>
internal virtual long GetSiteEventsRowCount() =>
_eventLogger.WithConnection(connection => ReadPragmaLong(connection, "SELECT COUNT(*) FROM site_events"));
/// <summary>
/// One purge tick's shared delete allowance, drawn down by the retention purge and then the
/// storage-cap trim.
src/ZB.MOM.WW.ScadaBridge.SiteEventLogging/SiteEventLogOptions.cs
+28 -1
View File
@@ -4,7 +4,34 @@ public class SiteEventLogOptions
{
/// <summary>Number of days to retain site event log entries before purge; default 30.</summary>
public int RetentionDays { get; set; } = 30;
/// <summary>Maximum SQLite database size in megabytes before old entries are purged; default 1024 MB.</summary>
/// <summary>
/// Maximum size in megabytes of <c>site_events</c> AND ITS FOUR INDEXES before the oldest
/// entries are trimmed; default 1024 MB.
/// </summary>
/// <remarks>
/// <para>
/// <b>This is not a cap on the database file</b> (register row 81, decision record
/// <c>docs/plans/2026-08-17-site-event-log-cap-semantics-decision.md</c>). <c>site_events</c>
/// shares one SQLite file at <c>LocalDb:Path</c> with nine other tables plus LocalDb's
/// <c>__localdb_oplog</c> / <c>__localdb_row_version</c> bookkeeping. Measuring the whole file
/// made the trim unwinnable on a replication-enabled node — each batch of deletes generated
/// an equal number of oplog rows and tombstones in the same file, so the measured size never
/// fell and the trim stopped 2.5x over cap. Worse, it made the event log's budget a function
/// of the replication peer's health, in the direction where a replication fault destroys
/// operational history.
/// </para>
/// <para>
/// Consequence for sizing a site's data volume: this value alone does not bound it. LocalDb
/// replication bookkeeping is separate overhead on the same disk, bounded by
/// <c>LocalDb:Replication:MaxOplogRows</c> and <c>TombstoneRetention</c>. Budget the sum.
/// </para>
/// <para>
/// The footprint is MODELLED from sampled row width rather than measured exactly: SQLite's
/// <c>dbstat</c> virtual table would give an exact per-table page count but is not compiled
/// into the bundled native library. Measured accuracy is −0.8 % to +12.8 %, deliberately
/// biased high. See <c>EventLogPurgeService.EstimateSiteEventsSize</c>.
/// </para>
/// </remarks>
public int MaxStorageMb { get; set; } = 1024;
/// <summary>
/// MIGRATION-ONLY as of LocalDb Phase 1. <see cref="SiteEventLogger"/> no longer reads