fix(localdb): root-cause the soak disk-I/O failure + ship the hardening follow-ups
The Phase 2 soak's "LocalDb fails under load" blocker is NOT a product defect. Root cause: host-side (macOS) sqlite3 reads of the live, bind-mounted WAL databases. POSIX advisory locks do not propagate across the virtiofs boundary, so the host reader believes it is the only connection, checkpoints on close and resets the WAL to 0 bytes under the container. The container's still-mapped WAL index then references frames that no longer exist and every subsequent statement fails SQLITE_IOERR_SHORT_READ (522) -> primary code 10, permanently until the process reopens the database. Reproduced on demand both on the rig (one sqlite3 SELECT reset a 4.6 MiB WAL and produced the first error one second later) and in a minimal python:3.12-alpine repro with no LocalDb or .NET involved. Refuted the converse: a freshly-reopened node sustained the full soak load 10+ minutes with zero errors. The original brief's isolation was confounded - both nodes had been poisoned by the same sampling pass, and a poisoned standby looks healthy only because it issues almost no statements. Corrections annotated in place. Hardening shipped alongside: - SqliteErrorCodes.Describe: log SQLite primary AND extended codes at the LocalDb-adjacent catch sites (the missing extended code is what made the original diagnosis so slow). - SiteAuditTelemetryActor: stop touching ActorContext across an await (NotSupportedException), with regression coverage. - infra/reseed.sh: apply the MSSQL init scripts explicitly. The official mssql/server image does not implement /docker-entrypoint-initdb.d, so a fresh volume hung the reseed forever; compose mounts annotated as informational. Deliberately NOT done: detect-and-reopen self-heal in SqliteLocalDb. It defends only against external interference, which is now prevented at the source, and same-kernel production readers see the locks correctly. Build 0 warnings; SiteAuditTelemetryActorTests 9/9, SiteEventLogging 70/70. Claude-Session: https://claude.ai/code/session_01BL2Vu1ESDQ9SCN4gVKkdts
This commit is contained in:
@@ -114,8 +114,8 @@ public sealed class CachedCallTelemetryForwarder : ICachedCallTelemetryForwarder
|
||||
// Kind/Status are domain fields carried in DetailsJson — decompose to log them.
|
||||
var d = AuditRowProjection.Decompose(telemetry.Audit);
|
||||
_logger.LogWarning(ex,
|
||||
"CachedCallTelemetryForwarder: audit emission threw for EventId {EventId} (Kind {Kind}, Status {Status})",
|
||||
d.EventId, d.Kind, d.Status);
|
||||
"CachedCallTelemetryForwarder: audit emission threw for EventId {EventId} (Kind {Kind}, Status {Status}, sqlite {SqliteError})",
|
||||
d.EventId, d.Kind, d.Status, SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,8 +192,8 @@ public sealed class CachedCallTelemetryForwarder : ICachedCallTelemetryForwarder
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"CachedCallTelemetryForwarder: tracking-store emission threw for TrackedOperationId {Id} (Status {Status})",
|
||||
telemetry.Operational.TrackedOperationId, telemetry.Operational.Status);
|
||||
"CachedCallTelemetryForwarder: tracking-store emission threw for TrackedOperationId {Id} (Status {Status}, sqlite {SqliteError})",
|
||||
telemetry.Operational.TrackedOperationId, telemetry.Operational.Status, SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +68,15 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
private readonly IOperationTrackingStore? _trackingStore;
|
||||
private readonly SiteAuditTelemetryOptions _options;
|
||||
private readonly ILogger<SiteAuditTelemetryActor> _logger;
|
||||
// Captured at construction (both are thread-safe immutable handles) because
|
||||
// ScheduleNext/ScheduleNextCached run from the drains' finally blocks, whose
|
||||
// ConfigureAwait(false) continuations complete on pool threads with no
|
||||
// active ActorContext — reading Context/Self there either throws
|
||||
// NotSupportedException or, worse, silently resolves a STALE cell left in
|
||||
// the thread-static slot and re-arms the tick at the wrong actor
|
||||
// (known-issues/2026-07-20-localdb-disk-io-error-under-load.md §8).
|
||||
private readonly IScheduler _scheduler;
|
||||
private readonly IActorRef _self;
|
||||
private ICancelable? _pendingTick;
|
||||
private ICancelable? _pendingCachedTick;
|
||||
// Per-actor lifecycle CTS so an in-flight drain (queue read,
|
||||
@@ -108,6 +117,8 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
_trackingStore = trackingStore;
|
||||
_scheduler = Context.System.Scheduler;
|
||||
_self = Self;
|
||||
|
||||
ReceiveAsync<Drain>(_ => OnDrainAsync());
|
||||
ReceiveAsync<CachedDrain>(_ => OnCachedDrainAsync());
|
||||
@@ -197,7 +208,9 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
{
|
||||
// Catch-all so a SQLite hiccup or mapper bug never crashes the
|
||||
// actor. The next tick is still scheduled in the finally block.
|
||||
_logger.LogError(ex, "Unexpected error during audit-log telemetry drain.");
|
||||
_logger.LogError(ex,
|
||||
"Unexpected error during audit-log telemetry drain (sqlite {SqliteError}).",
|
||||
SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -278,8 +291,8 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
// batch — the audit half is best-effort. Log and skip
|
||||
// this row; it stays Pending for the next drain.
|
||||
_logger.LogWarning(ex,
|
||||
"Cached-telemetry drain: tracking lookup threw for {EventId} (TrackedOperationId {Tid}); skipping.",
|
||||
auditRow.EventId, auditRow.CorrelationId);
|
||||
"Cached-telemetry drain: tracking lookup threw for {EventId} (TrackedOperationId {Tid}, sqlite {SqliteError}); skipping.",
|
||||
auditRow.EventId, auditRow.CorrelationId, SqliteErrorCodes.Describe(ex));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -332,7 +345,9 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unexpected error during cached-telemetry drain.");
|
||||
_logger.LogError(ex,
|
||||
"Unexpected error during cached-telemetry drain (sqlite {SqliteError}).",
|
||||
SqliteErrorCodes.Describe(ex));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -428,24 +443,26 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
return list;
|
||||
}
|
||||
|
||||
// Must stay off Context/Self: called from off-context continuations — see
|
||||
// the _scheduler/_self field comment.
|
||||
private void ScheduleNext(TimeSpan delay)
|
||||
{
|
||||
_pendingTick?.Cancel();
|
||||
_pendingTick = Context.System.Scheduler.ScheduleTellOnceCancelable(
|
||||
_pendingTick = _scheduler.ScheduleTellOnceCancelable(
|
||||
delay,
|
||||
Self,
|
||||
_self,
|
||||
Drain.Instance,
|
||||
Self);
|
||||
_self);
|
||||
}
|
||||
|
||||
private void ScheduleNextCached(TimeSpan delay)
|
||||
{
|
||||
_pendingCachedTick?.Cancel();
|
||||
_pendingCachedTick = Context.System.Scheduler.ScheduleTellOnceCancelable(
|
||||
_pendingCachedTick = _scheduler.ScheduleTellOnceCancelable(
|
||||
delay,
|
||||
Self,
|
||||
_self,
|
||||
CachedDrain.Instance,
|
||||
Self);
|
||||
_self);
|
||||
}
|
||||
|
||||
/// <summary>Self-tick message that triggers an audit-only drain cycle.</summary>
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Renders the primary/extended SQLite result codes of a
|
||||
/// <see cref="SqliteException"/> for log messages. The exception's own message
|
||||
/// carries only the primary code ("SQLite Error 10: 'disk I/O error'"), which
|
||||
/// is too generic to act on — the 2026-07-20 disk-I/O incident
|
||||
/// (known-issues/2026-07-20-localdb-disk-io-error-under-load.md) had to be
|
||||
/// reproduced from scratch to learn the extended code (522 =
|
||||
/// SQLITE_IOERR_SHORT_READ) that names the failing operation.
|
||||
/// </summary>
|
||||
internal static class SqliteErrorCodes
|
||||
{
|
||||
/// <summary>
|
||||
/// "primary/extended" (e.g. "10/522") for a <see cref="SqliteException"/>
|
||||
/// anywhere in the exception chain; "n/a" for non-SQLite failures.
|
||||
/// </summary>
|
||||
public static string Describe(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
{
|
||||
if (e is SqliteException se)
|
||||
{
|
||||
return $"{se.SqliteErrorCode}/{se.SqliteExtendedErrorCode}";
|
||||
}
|
||||
}
|
||||
return "n/a";
|
||||
}
|
||||
}
|
||||
@@ -257,8 +257,8 @@ public class SiteEventLogger : ISiteEventLogger, IDisposable
|
||||
// (Health Monitoring reads FailedWriteCount) and fault the caller's
|
||||
// Task instead of silently discarding the exception.
|
||||
Interlocked.Increment(ref _failedWriteCount);
|
||||
_logger.LogError(ex, "Failed to record event: {EventType} from {Source}",
|
||||
pending.EventType, pending.Source);
|
||||
_logger.LogError(ex, "Failed to record event: {EventType} from {Source} (sqlite {SqliteError})",
|
||||
pending.EventType, pending.Source, DescribeSqliteError(ex));
|
||||
pending.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
@@ -297,6 +297,25 @@ public class SiteEventLogger : ISiteEventLogger, IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// "primary/extended" SQLite result codes (e.g. "10/522") for a
|
||||
/// <see cref="SqliteException"/> anywhere in the chain; "n/a" otherwise.
|
||||
/// The exception message alone carries only the primary code, which proved
|
||||
/// too generic to diagnose the 2026-07-20 disk-I/O incident
|
||||
/// (known-issues/2026-07-20-localdb-disk-io-error-under-load.md).
|
||||
/// </summary>
|
||||
private static string DescribeSqliteError(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
{
|
||||
if (e is SqliteException se)
|
||||
{
|
||||
return $"{se.SqliteErrorCode}/{se.SqliteExtendedErrorCode}";
|
||||
}
|
||||
}
|
||||
return "n/a";
|
||||
}
|
||||
|
||||
/// <summary>An event awaiting persistence by the background writer.</summary>
|
||||
private sealed record PendingEvent(
|
||||
string Id,
|
||||
|
||||
Reference in New Issue
Block a user