fix(runtime): fast-fail writes in degraded driver states + swallow self SubscriptionEstablished

This commit is contained in:
Joseph Doherty
2026-06-14 00:34:37 -04:00
parent 99eea0b455
commit 42b4a923fd
2 changed files with 298 additions and 0 deletions
@@ -211,6 +211,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
private void Connecting()
{
Receive<InitializeRequested>(msg => InitializeAsync(msg.DriverConfigJson));
// Fast-fail writes while still connecting — without this the inbound WriteAttribute dead-letters
// and DriverHostActor.HandleRouteNodeWrite waits its full 8s Ask before reporting a generic
// "write timeout". Synchronous Receive: Sender.Tell on the actor thread is safe (#4a-instance).
Receive<WriteAttribute>(_ =>
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
Receive<InitializeSucceeded>(_ =>
{
_log.Info("DriverInstance {Id}: connected", _driverInstanceId);
@@ -227,6 +232,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
});
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
Receive<ForceReconnect>(_ => { /* already connecting — no-op */ });
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
Receive<SubscriptionEstablished>(msg =>
_log.Debug("DriverInstance {Id}: subscription established ({Count} refs, {Diag})",
_driverInstanceId, msg.ReferenceCount, msg.DiagnosticId));
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
}
@@ -259,12 +269,21 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
else if (_subscriptionHandle is not null) Self.Tell(new Unsubscribe());
});
Receive<DataChangeForward>(OnDataChangeForward);
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
Receive<SubscriptionEstablished>(msg =>
_log.Debug("DriverInstance {Id}: subscription established ({Count} refs, {Diag})",
_driverInstanceId, msg.ReferenceCount, msg.DiagnosticId));
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
}
private void Reconnecting()
{
Receive<RetryConnect>(_ => InitializeAsync(_currentConfigJson ?? "{}"));
// Fast-fail writes while reconnecting (same reason as Connecting — avoids the 8s host Ask
// timeout on an inbound write to a transiently-down driver). Synchronous Receive (#4a-instance).
Receive<WriteAttribute>(_ =>
Sender.Tell(new WriteAttributeResult(false, "driver not connected")));
Receive<InitializeSucceeded>(_ =>
{
Timers.Cancel("retry-connect");
@@ -276,6 +295,11 @@ public sealed class DriverInstanceActor : ReceiveActor, IWithTimers
Receive<InitializeFailed>(_ => { /* keep retrying via timer */ });
Receive<SetDesiredSubscriptions>(StoreDesiredSubscriptions);
Receive<ForceReconnect>(_ => { /* already reconnecting — no-op */ });
// ResubscribeDesired self-Tells Subscribe; HandleSubscribeAsync replies SubscriptionEstablished to the
// sender, which on the self-resubscribe path is Self. Swallow it (trace only) so it doesn't dead-letter.
Receive<SubscriptionEstablished>(msg =>
_log.Debug("DriverInstance {Id}: subscription established ({Count} refs, {Diag})",
_driverInstanceId, msg.ReferenceCount, msg.DiagnosticId));
Receive<HealthPollTick>(_ => PublishHealthSnapshot());
Timers.StartPeriodicTimer("retry-connect", RetryConnect.Instance, _reconnectInterval);
}