fix(runtime): stash dedup + applying-wedge watchdog; cache-failure diagnostics

This commit is contained in:
Joseph Doherty
2026-08-15 17:43:15 -04:00
parent bfbdce32e4
commit 0dbbb5ac6b
3 changed files with 451 additions and 37 deletions
@@ -1,5 +1,6 @@
using System.Text.Json;
using Akka.Actor;
using Akka.Event;
using Microsoft.EntityFrameworkCore;
using Shouldly;
using Xunit;
@@ -93,11 +94,20 @@ public sealed class DriverHostActorAsyncCacheWriteTests()
localRoles: new HashSet<string> { "driver" },
deploymentArtifactCache: new AsyncFailingArtifactCache()));
EventFilter.Warning(contains: "could not be cached").ExpectOne(TimeSpan.FromSeconds(5), () =>
{
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
});
// Matched on the EXCEPTION, not just the prose: the pipe-back carries the Exception itself and
// the handler logs it as the event's Cause. A SQLite failure is routinely a wrapped inner
// exception whose outer Message names neither the result code nor the failing frame, so a
// Reason-string-only carrier would have thrown that away before the log line was written.
// (Custom rather than EventFilter.Exception<T>, which only ever matches ERROR-level events —
// a failed cache write is deliberately a Warning, since the apply itself succeeded.)
EventFilter.Custom<Warning>(e =>
e.Cause is InvalidOperationException &&
e.Message?.ToString()?.Contains("could not be cached", StringComparison.Ordinal) == true)
.ExpectOne(TimeSpan.FromSeconds(5), () =>
{
actor.Tell(new DispatchDeployment(deploymentId, RevA, CorrelationId.NewId()));
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).Outcome.ShouldBe(ApplyAckOutcome.Applied);
});
coordinator.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
}