fix: outbox KPI per-status legs, composite event-log indexes, AuditLog ingest FORCESEEK (#122, #123, #124)

#122: the Notification Outbox KPI snapshots ORed every status in one
aggregation, which scanned the clustered index once the table held ~1 M
Parked rows (0.7-2.6 s; Health dashboard up to 5 s). Each snapshot now runs
three status-equality legs that seek their own index (~0.12 s); a cold Health
load on the rig is 0.12-0.52 s.

#123: site_events event_type/severity indexes are now composites with
timestamp, so a filtered page is read in order and stops at the limit
(severity=Error over 1 GB: 2,287 -> 15 ms cold, 115 -> 1.6 ms warm). Costs
454 -> 530 B/row, ~14 % fewer rows under the same cap. The schema apply drops
the old indexes in place; the cap size model samples timestamp three times
and re-scores inside its band with unchanged constants.

#124: the set-based AuditLog insert's NOT EXISTS probe scanned the EventId
index and concurrent writers deadlocked on S/IX page locks. WITH (FORCESEEK):
4 writers 775 rows/s + 76 deadlocks in 30 s -> 5,214 rows/s + 0. New
concurrent-writer regression test fails 4/4 without the hint.
This commit is contained in:
Joseph Doherty
2026-11-10 16:46:42 -05:00
parent 2b80a6defa
commit 424db1555d
12 changed files with 386 additions and 158 deletions
@@ -47,7 +47,7 @@ public class SchemaIndexTests : IDisposable
using var reader = cmd.ExecuteReader();
while (reader.Read()) indexes.Add(reader.GetString(0));
Assert.Contains("idx_events_severity", indexes);
Assert.Contains("idx_events_severity_timestamp", indexes);
}
/// <summary>Verifies that severity filtered query, when uses index, not full scan.</summary>
@@ -70,7 +70,75 @@ public class SchemaIndexTests : IDisposable
}
var planText = plan.ToString();
Assert.Contains("idx_events_severity", planText);
Assert.Contains("idx_events_severity_timestamp", planText);
Assert.DoesNotContain("SCAN site_events", planText);
}
/// <summary>
/// Verifies that the query service's own page shape — an equality filter, ordered by
/// (timestamp, id), limited to a page — is answered in index order rather than by
/// sorting every matching row (#123).
/// </summary>
/// <param name="column">The equality-filtered column.</param>
/// <param name="expectedIndex">The composite index expected to answer it.</param>
[Theory]
[InlineData("severity", "idx_events_severity_timestamp")]
[InlineData("event_type", "idx_events_type_timestamp")]
public void EqualityFilteredPage_ReadsInTimestampOrder_WithoutSortingEveryMatch(string column, string expectedIndex)
{
foreach (var window in new[] { "", "AND timestamp >= '2026-01-01' " })
{
using var cmd = _verifyConnection.CreateCommand();
cmd.CommandText =
$"EXPLAIN QUERY PLAN SELECT id FROM site_events WHERE {column} = 'x' {window}" +
"ORDER BY timestamp ASC, id ASC LIMIT 501";
var plan = new System.Text.StringBuilder();
using var reader = cmd.ExecuteReader();
while (reader.Read())
plan.Append(reader.GetString(reader.GetOrdinal("detail"))).Append('\n');
var planText = plan.ToString();
Assert.Contains(expectedIndex, planText);
// "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" (ties on timestamp, broken by id)
// is fine; a whole-ORDER-BY temp b-tree means every match is read and sorted.
Assert.DoesNotContain("USE TEMP B-TREE FOR ORDER BY", planText);
}
}
/// <summary>Verifies that a pre-#123 file loses its single-column type/severity indexes on the next schema apply.</summary>
[Fact]
public void Apply_OnAPre123File_ReplacesTheSingleColumnIndexes()
{
using var connection = new SqliteConnection("Data Source=:memory:");
connection.Open();
using (var cmd = connection.CreateCommand())
{
cmd.CommandText = """
CREATE TABLE site_events (id TEXT NOT NULL PRIMARY KEY, timestamp TEXT NOT NULL,
event_type TEXT NOT NULL, severity TEXT NOT NULL, instance_id TEXT,
source TEXT NOT NULL, message TEXT NOT NULL, details TEXT);
CREATE INDEX idx_events_timestamp ON site_events(timestamp);
CREATE INDEX idx_events_type ON site_events(event_type);
CREATE INDEX idx_events_instance ON site_events(instance_id);
CREATE INDEX idx_events_severity ON site_events(severity);
INSERT INTO site_events VALUES ('a', '2026-01-01T00:00:00.0000000+00:00', 'alarm', 'Info', NULL, 's', 'm', NULL);
""";
cmd.ExecuteNonQuery();
}
SiteEventLogSchema.Apply(connection);
using var list = connection.CreateCommand();
list.CommandText = "SELECT name FROM pragma_index_list('site_events') WHERE origin = 'c' ORDER BY name";
var indexes = new List<string>();
using (var reader = list.ExecuteReader())
while (reader.Read()) indexes.Add(reader.GetString(0));
Assert.Equal(
["idx_events_instance", "idx_events_severity_timestamp", "idx_events_timestamp", "idx_events_type_timestamp"],
indexes);
using var count = connection.CreateCommand();
count.CommandText = "SELECT COUNT(*) FROM site_events INDEXED BY idx_events_severity_timestamp WHERE severity = 'Info'";
Assert.Equal(1L, (long)count.ExecuteScalar()!);
}
}
@@ -121,7 +121,7 @@ public class SiteEventLogSchemaTests
while (reader.Read()) indexes.Add(reader.GetString(0));
Assert.Equal(
["idx_events_instance", "idx_events_severity", "idx_events_timestamp", "idx_events_type"],
["idx_events_instance", "idx_events_severity_timestamp", "idx_events_timestamp", "idx_events_type_timestamp"],
indexes);
}
@@ -76,35 +76,36 @@ public class SiteEventsSchemaCalibrationTests
indexes.SequenceEqual(
[
"idx_events_instance",
"idx_events_severity",
"idx_events_severity_timestamp",
"idx_events_timestamp",
"idx_events_type",
"idx_events_type_timestamp",
"sqlite_autoindex_site_events_1",
]),
$"{RecalibrationNotice} Actual indexes: {string.Join(", ", indexes)}");
}
/// <summary>Verifies that each index covers exactly the column the sampler sums.</summary>
/// <summary>Verifies that each index covers exactly the columns the sampler sums for it.</summary>
/// <param name="indexName">The index name.</param>
/// <param name="expectedColumn">The column the index is expected to cover.</param>
/// <param name="expectedColumns">The columns the index is expected to cover, comma-separated, in key order.</param>
[Theory]
[InlineData("idx_events_timestamp", "timestamp")]
[InlineData("idx_events_type", "event_type")]
[InlineData("idx_events_instance", "instance_id")]
[InlineData("idx_events_severity", "severity")]
public void EachIndex_CoversExactlyTheColumnTheSamplerSums(string indexName, string expectedColumn)
[InlineData("idx_events_type_timestamp", "event_type,timestamp")]
[InlineData("idx_events_severity_timestamp", "severity,timestamp")]
public void EachIndex_CoversExactlyTheColumnsTheSamplerSums(string indexName, string expectedColumns)
{
// The sampler's indexed_bytes expression sums octet_length(timestamp) +
// octet_length(event_type) + octet_length(severity) + octet_length(instance_id) —
// one column per index. Widening an index to a composite (or re-pointing it) would
// change the real entry width without changing what is sampled.
// The sampler's indexed_bytes expression sums 3 * octet_length(timestamp) +
// octet_length(event_type) + octet_length(severity) + octet_length(instance_id):
// timestamp once for its own index and once for each of the two composites (#123).
// Widening or re-pointing an index changes the real entry width without changing
// what is sampled.
using var connection = OpenWithSchema();
var columns = ReadStrings(
connection, $"SELECT name FROM pragma_index_info('{indexName}')");
connection, $"SELECT name FROM pragma_index_info('{indexName}') ORDER BY seqno");
Assert.True(
columns.SequenceEqual([expectedColumn]),
columns.SequenceEqual(expectedColumns.Split(',')),
$"{RecalibrationNotice} Index {indexName} covers: {string.Join(", ", columns)}");
}
}