fix(runtime): malformed artifact bytes on boot/restore are no answer (#601); evict telemetry snapshots on driver removal (#609)

Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
Joseph Doherty
2026-10-08 06:53:45 -04:00
parent 3baa332d6b
commit 53247a056a
10 changed files with 162 additions and 6 deletions
@@ -1,4 +1,5 @@
using System.Diagnostics;
using System.Text.Json;
using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
@@ -1097,7 +1098,12 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
{
var correlation = CorrelationId.NewId();
ReconcileDriversFromBlob(deploymentId, blob);
// #601: null = the cached bytes were empty or malformed — no answer, not an empty configuration.
// Keep last-known-good: no address-space rebuild and no subscription push from unusable bytes.
// #601: null = the cached bytes were empty or malformed — no answer, not an empty configuration.
// Keep last-known-good: no address-space rebuild and no subscription push from unusable bytes.
if (ReconcileDriversFromBlob(deploymentId, blob) is null)
return;
// Hand the cached blob to the rebuild so it materialises the client-facing address space from
// it directly. Passing only the deploymentId would drive a ConfigDb read — unreachable here by
@@ -2483,7 +2489,25 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
return null;
}
ApplyDriverSpecs(DeploymentArtifact.ParseDriverInstances(blob, _localNode.Value));
// #601 (S-10): non-empty but malformed bytes are "no answer" too. The lenient static
// ParseDriverInstances swallows the JsonException into an empty list, and an empty list plans every
// running child for StopChild. Open throws instead, so a corrupt cache row / restored blob leaves
// the running drivers alone, exactly like the zero-length case above.
IReadOnlyList<DriverInstanceSpec> specs;
try
{
using var parsed = DeploymentArtifact.Open(blob);
specs = parsed.ParseDriverInstances(_localNode.Value);
}
catch (JsonException ex)
{
_log.Error(ex,
"DriverHost {Node}: artifact for {Id} is malformed ({Bytes} bytes); skipping reconcile and keeping the {Count} running driver(s)",
_localNode, deploymentId, blob.Length, _children.Count);
return null;
}
ApplyDriverSpecs(specs);
return blob;
}
@@ -2772,10 +2796,14 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
return;
}
// #601 (S-10): Open, not the lenient static ParseComposition — that wrapper swallows malformed JSON
// into an EMPTY composition, which would hand every child an empty desired set and drop its live
// subscription. A malformed artifact is no answer: leave the subscriptions as they are.
AddressSpaceComposition composition;
try
{
composition = DeploymentArtifact.ParseComposition(blob, _localNode.Value);
using var parsed = DeploymentArtifact.Open(blob);
composition = parsed.ParseComposition(_localNode.Value);
}
catch (Exception ex)
{
@@ -3204,6 +3232,9 @@ public sealed class DriverHostActor : ReceiveActor, IWithTimers, IWithUnboundedS
StopDependencyAdapter(driverInstanceId);
Context.Stop(entry.Actor);
_children.Remove(driverInstanceId);
// #609: the instance left the plan (StopChild is only reached via plan.ToStop), so its cached
// Health/Resilience snapshots must not replay to new telemetry subscribers.
_telemetryHub?.Evict(driverInstanceId);
_log.Info("DriverHost {Node}: stopped driver child {Id}", _localNode, driverInstanceId);
}