fix: outbox KPI per-status legs, composite event-log indexes, AuditLog ingest FORCESEEK (#122, #123, #124)

#122: the Notification Outbox KPI snapshots ORed every status in one
aggregation, which scanned the clustered index once the table held ~1 M
Parked rows (0.7-2.6 s; Health dashboard up to 5 s). Each snapshot now runs
three status-equality legs that seek their own index (~0.12 s); a cold Health
load on the rig is 0.12-0.52 s.

#123: site_events event_type/severity indexes are now composites with
timestamp, so a filtered page is read in order and stops at the limit
(severity=Error over 1 GB: 2,287 -> 15 ms cold, 115 -> 1.6 ms warm). Costs
454 -> 530 B/row, ~14 % fewer rows under the same cap. The schema apply drops
the old indexes in place; the cap size model samples timestamp three times
and re-scores inside its band with unchanged constants.

#124: the set-based AuditLog insert's NOT EXISTS probe scanned the EventId
index and concurrent writers deadlocked on S/IX page locks. WITH (FORCESEEK):
4 writers 775 rows/s + 76 deadlocks in 30 s -> 5,214 rows/s + 0. New
concurrent-writer regression test fails 4/4 without the hint.
This commit is contained in:
Joseph Doherty
2026-11-10 16:46:42 -05:00
parent 2b80a6defa
commit 424db1555d
12 changed files with 386 additions and 158 deletions
@@ -89,7 +89,11 @@ public class EventLogPurgeService : BackgroundService
/// </summary>
private const double TablePageFillFactor = 0.78;
/// <summary>Number of indexes on <c>site_events</c> (timestamp, event_type, instance_id, severity).</summary>
/// <summary>
/// Number of secondary indexes on <c>site_events</c>: timestamp, instance_id,
/// (event_type, timestamp) and (severity, timestamp). The sampled <c>indexed_bytes</c>
/// charges every covered column once per index, hence <c>3 * octet_length(timestamp)</c>.
/// </summary>
private const int IndexedColumnCount = 4;
/// <summary>Per-entry bookkeeping in an index B-tree leaf: key-length varint, rowid and cell pointer.</summary>
@@ -680,8 +684,8 @@ public class EventLogPurgeService : BackgroundService
/// <para>
/// <b>The model.</b> Rows are sampled from both ends of the table (<see cref="SizeSampleRows"/>
/// from each — the head is what the oldest-first trim will actually delete, the tail catches a
/// workload whose event shape has drifted) to get an average payload width and an average
/// width across the four indexed columns. Table cost is then computed as whole pages —
/// workload whose event shape has drifted) to get an average payload width and the average
/// summed width of the columns each of the four indexes covers. Table cost is then computed as whole pages —
/// <c>ceil(rows / rowsPerPage) × pageSize</c> — rather than as a flat bytes-per-row rate,
/// because that is the term that dominates once rows approach half a page: rows too wide to
/// pack two-per-page waste the remainder, which a linear model underestimates by ~40 %.
@@ -730,7 +734,7 @@ public class EventLogPurgeService : BackgroundService
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
+ octet_length(source) + octet_length(message)
+ COALESCE(octet_length(details), 0) AS payload_bytes,
octet_length(timestamp) + octet_length(event_type)
(3 * octet_length(timestamp)) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
AS indexed_bytes
FROM site_events LIMIT {SizeSampleRows}
@@ -742,7 +746,7 @@ public class EventLogPurgeService : BackgroundService
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
+ octet_length(source) + octet_length(message)
+ COALESCE(octet_length(details), 0) AS payload_bytes,
octet_length(timestamp) + octet_length(event_type)
(3 * octet_length(timestamp)) + octet_length(event_type)
+ octet_length(severity) + COALESCE(octet_length(instance_id), 0)
AS indexed_bytes
FROM site_events ORDER BY rowid DESC LIMIT {SizeSampleRows}
@@ -67,10 +67,24 @@ public static class SiteEventLogSchema
details TEXT
);
CREATE INDEX IF NOT EXISTS idx_events_timestamp ON site_events(timestamp);
CREATE INDEX IF NOT EXISTS idx_events_type ON site_events(event_type);
CREATE INDEX IF NOT EXISTS idx_events_instance ON site_events(instance_id);
CREATE INDEX IF NOT EXISTS idx_events_severity ON site_events(severity);
DROP INDEX IF EXISTS idx_events_type;
DROP INDEX IF EXISTS idx_events_severity;
CREATE INDEX IF NOT EXISTS idx_events_type_timestamp ON site_events(event_type, timestamp);
CREATE INDEX IF NOT EXISTS idx_events_severity_timestamp ON site_events(severity, timestamp);
""";
// The type and severity indexes are composites with timestamp (#123). The query
// service always orders by (timestamp, id) and returns one page, so a
// single-column index made an equality filter read EVERY matching row and sort
// it before returning 500 (severity=Error over a full 1 GB log: 2.3 s cold,
// 115 ms warm). The composite returns the rows already in timestamp order and
// stops at the page limit (~1 ms). The cost is ~69 bytes per row more index,
// which the storage-cap size model charges (EventLogPurgeService sampling SQL).
// The DROPs migrate a pre-#123 file in place on its first boot; dropping and
// building an index on a 1 GB table takes seconds and runs once. Indexes are not
// part of LocalDb's replicated-table digest, so a pair may upgrade one node at a
// time as far as this change is concerned.
//
// The query service also supports keyword search via leading-wildcard
// LIKE on message/source. A leading-wildcard LIKE cannot use a B-tree
// index, so that path intentionally full-scans; severity/event_type/