fix(runtime): a removed driver's late writes no longer resurrect its resilience/telemetry state (#609)
DriverHostActor.StopChild released the tracker (Invalidate) and evicted the hub synchronously, but Context.Stop is asynchronous: the stopping child's in-flight SubscribeBulk completed through its capability invoker AFTER the release, and its RecordCallStart/RecordSuccess/RecordCallComplete re-created the tracker entry, which DriverResilienceStatusPublisherService then re-emitted every tick forever. Confirmed with a temporary stack capture in the tracker (removed): every post-Release write came from DriverInstanceActor.HandleSubscribeAsync -> CapabilityInvoker.ExecuteAsync; the test failed whenever it landed before the second publisher tick. - DriverResilienceStatusTracker.Retire (Release) drops the id's snapshots and ignores every later Record* for it; one guarded write path re-checks after the upsert so a write racing Retire cannot survive it. Reinstate (called by DriverCapabilityInvokerFactory.Create) lifts it, so the same id re-added in a later deployment records again. - TelemetryLocalHub.Evict now also holds the id (late Health/Resilience dropped, neither cached nor fanned out); ITelemetryLocalHub.Readmit (default no-op), called at the top of SpawnChild, lifts it. - The removal test is now deterministic: an invoker decorator holds the child's subscribe inside the real invoker's bracket across the removal and signals when its late writes are done (positive control) — red 5/5 without the fix, green 20/20 with it. New unit tests for the tracker guard (incl. a Retire race), the factory Release/Create pair, the hub hold/readmit, and a re-add-same-id host test. Claude-Session: https://claude.ai/code/session_01Ef2ozxS7QPanGT8uoATbTe
This commit is contained in:
@@ -42,6 +42,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
private readonly ConcurrentDictionary<string, TelemetryItem.Health> _healthCache = new();
|
||||
private readonly ConcurrentDictionary<(string InstanceId, string HostName), TelemetryItem.Resilience> _resilienceCache = new();
|
||||
|
||||
// Instances evicted and not since readmitted (#609). Their late Health/Resilience items are dropped, so a
|
||||
// removed driver's still-stopping child cannot re-prime the cache it was just evicted from. Under _gate.
|
||||
private readonly HashSet<string> _evicted = new(StringComparer.Ordinal);
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Emit(TelemetryItem item)
|
||||
{
|
||||
@@ -51,6 +55,10 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
{
|
||||
switch (item)
|
||||
{
|
||||
case TelemetryItem.Health h when _evicted.Contains(h.E.DriverInstanceId):
|
||||
return;
|
||||
case TelemetryItem.Resilience r when _evicted.Contains(r.E.DriverInstanceId):
|
||||
return;
|
||||
case TelemetryItem.Health h:
|
||||
_healthCache[h.E.DriverInstanceId] = h;
|
||||
break;
|
||||
@@ -72,6 +80,7 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
_evicted.Add(driverInstanceId);
|
||||
_healthCache.TryRemove(driverInstanceId, out _);
|
||||
foreach (var key in _resilienceCache.Keys)
|
||||
if (string.Equals(key.InstanceId, driverInstanceId, StringComparison.Ordinal))
|
||||
@@ -79,6 +88,15 @@ public sealed class TelemetryLocalHub : ITelemetryLocalHub
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Readmit(string driverInstanceId)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(driverInstanceId);
|
||||
|
||||
lock (_gate)
|
||||
_evicted.Remove(driverInstanceId);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ITelemetrySubscription Subscribe(int boundedCapacity)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user