fix(galaxy): probe advises start the event stream first so initial ScanState arrives (#865)

The gateway's StreamEvents with after_worker_sequence = 0 is a fresh stream
and never replays, and the event pump started only on the first
Read/Subscribe, so platform ScanState probes advised before then lost their
initial value (and a driver with no authored tags never got any host status).

PerPlatformProbeWatcher now awaits an ensure-event-stream hook before every
probe SubscribeBulk (seed, discover sync, reconnect ResubscribeStaleAsync).
The driver's hook starts the pump (refused while a teardown is in progress)
and waits, bounded, on the new EventPump.StreamRequested signal - completed
once the stream loop's first MoveNext has issued the StreamEvents RPC, or
when the loop ends - so the advise never precedes the stream request.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
Joseph Doherty
2026-11-11 12:24:08 -05:00
parent c4bc1ef309
commit e35f2a4e93
5 changed files with 320 additions and 4 deletions
@@ -540,6 +540,9 @@ public sealed class GalaxyDriver
// a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It
// runs in the background: InitializeAsync neither waits on it nor fails because of it.
await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false);
// #865: the seed's probe advises start the event pump themselves (BuildProbeWatcher's
// pre-subscribe hook) — a driver with no authored tags never Reads/Subscribes, and the pump
// used to start only there, so every initial ScanState value went to no stream.
RequestProbeSeed("initialize");
_logger.LogInformation(
"GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
@@ -730,9 +733,12 @@ public sealed class GalaxyDriver
// The watcher publishes its probe handles into the driver's registry (reserved id) so the
// EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every
// probe event was dropped before it got anywhere near the watcher.
// #865: the watcher also starts the pump before every probe SubscribeBulk — the initial ScanState
// value only reaches a stream that is already attached (the gateway does not replay a fresh one).
private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
new(subscriber, _hostStatuses, _subscriptions, _logger,
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs,
ensureEventStream: EnsureProbeEventStreamAsync);
/// <summary>
/// Build the production gw client + session + per-capability runtime components
@@ -2101,6 +2107,47 @@ public sealed class GalaxyDriver
}
}
/// <summary>
/// <see cref="EnsureEventPumpStarted"/> for infrastructure callers (the probe watcher) that
/// must not fail because the runtime is going away: false — and no pump — while a teardown is in
/// progress or there is no subscriber.
/// </summary>
/// <returns>The running (or just started) pump; null when none may be started.</returns>
private EventPump? TryEnsureEventPumpStarted()
{
lock (_pumpLock)
{
if (_runtimeStopping || _disposed || _subscriber is null) return null;
return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant); cannot throw here
}
}
/// <summary>Upper bound on how long a probe subscribe waits for the pump's StreamEvents call to be issued.</summary>
internal TimeSpan ProbeStreamRequestWait { get; set; } = TimeSpan.FromSeconds(5);
/// <summary>
/// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait,
/// bounded, until its StreamEvents call has been issued, so the gateway has a subscriber to push
/// the probe's initial ScanState value to. A fresh stream (after_worker_sequence = 0) is never
/// replayed, so an initial value pushed before it attaches is lost — and ScanState rarely changes
/// again. On timeout the subscribe proceeds anyway (logged): a late stream is no worse than before.
/// </summary>
private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken)
{
var pump = TryEnsureEventPumpStarted();
if (pump is null) return;
try
{
await pump.StreamRequested.WaitAsync(ProbeStreamRequestWait, cancellationToken).ConfigureAwait(false);
}
catch (TimeoutException)
{
_logger.LogWarning(
"GalaxyDriver {InstanceId} event stream was not requested within {Wait} — subscribing the platform probes anyway; their initial ScanState may be missed.",
_driverInstanceId, ProbeStreamRequestWait);
}
}
/// <summary>
/// Replay-gap callback for the <see cref="EventPump"/>. The gateway told us it evicted
/// events we can never receive, so every cached value may be stale by an unknown amount.
@@ -44,6 +44,11 @@ public sealed class PerPlatformProbeWatcher : IDisposable
// driven directly (unit tests of the decoding table).
private readonly SubscriptionRegistry? _registry;
// #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the
// gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events,
// and ScanState rarely changes — a lost initial value leaves the host missing indefinitely).
private readonly Func<CancellationToken, Task>? _ensureEventStream;
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
@@ -79,16 +84,23 @@ public sealed class PerPlatformProbeWatcher : IDisposable
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
/// <param name="logger">Optional logger for diagnostic messages.</param>
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
/// <param name="ensureEventStream">
/// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries
/// the initial ScanState value is running and requested (the driver starts its
/// <see cref="EventPump"/> and waits for its StreamEvents call to be issued).
/// </param>
internal PerPlatformProbeWatcher(
IGalaxySubscriber subscriber,
HostStatusAggregator aggregator,
SubscriptionRegistry? registry,
ILogger? logger = null,
int bufferedUpdateIntervalMs = 0)
int bufferedUpdateIntervalMs = 0,
Func<CancellationToken, Task>? ensureEventStream = null)
{
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
_registry = registry;
_ensureEventStream = ensureEventStream;
_logger = logger ?? NullLogger.Instance;
if (bufferedUpdateIntervalMs < 0)
{
@@ -196,6 +208,10 @@ public sealed class PerPlatformProbeWatcher : IDisposable
{
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
// #865: the stream must be live BEFORE the advise — the initial value is the probe's only
// event until the platform's scan state changes, and an unattached stream never sees it.
if (_ensureEventStream is not null) await _ensureEventStream(cancellationToken).ConfigureAwait(false);
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
// PublishBindingsLocked registers the handles below (window closes at method exit).
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
@@ -148,6 +148,17 @@ internal sealed class EventPump : IAsyncDisposable
private readonly int _session;
private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>
/// Completes once the stream loop has issued its <c>StreamEvents</c> call (the first
/// <c>MoveNextAsync</c> runs the subscriber's iterator synchronously up to the RPC) — or once the
/// loop has ended without getting that far, so a waiter never hangs on a dead pump. #865: the
/// gateway does not replay a fresh stream, so a caller whose only expected event is the initial
/// value of an advise it is about to make waits on this first.
/// </summary>
public Task StreamRequested => _streamRequested.Task;
/// <summary>
/// Marker written into the channel to wake the dispatch loop after a registration promoted
/// parked early events (#859). Never dispatched itself; compared by reference.
@@ -179,10 +190,17 @@ internal sealed class EventPump : IAsyncDisposable
private async Task RunAsync(CancellationToken ct)
{
IAsyncEnumerator<MxEvent>? enumerator = null;
try
{
await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false))
enumerator = _subscriber.StreamEventsAsync(ct).GetAsyncEnumerator(ct);
// Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is
// the moment StreamRequested reports (#865).
var next = enumerator.MoveNextAsync();
_streamRequested.TrySetResult();
for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync())
{
var ev = enumerator.Current;
if (ct.IsCancellationRequested) break;
EventsReceived.Add(1, _clientTag);
@@ -241,6 +259,20 @@ internal sealed class EventPump : IAsyncDisposable
}
finally
{
// A loop that died before issuing the RPC must not strand a StreamRequested waiter.
_streamRequested.TrySetResult();
if (enumerator is not null)
{
try { await enumerator.DisposeAsync().ConfigureAwait(false); }
catch (Exception ex) when (ct.IsCancellationRequested || ex is OperationCanceledException)
{
// Disposing a cancelled stream may surface its cancellation — shutdown noise.
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Galaxy EventPump stream enumerator dispose failed.");
}
}
// Tell the dispatch loop the producer is done so it drains and exits.
_channel.Writer.TryComplete();
}