fix(galaxy): probe advises start the event stream first so initial ScanState arrives (#865)
The gateway's StreamEvents with after_worker_sequence = 0 is a fresh stream and never replays, and the event pump started only on the first Read/Subscribe, so platform ScanState probes advised before then lost their initial value (and a driver with no authored tags never got any host status). PerPlatformProbeWatcher now awaits an ensure-event-stream hook before every probe SubscribeBulk (seed, discover sync, reconnect ResubscribeStaleAsync). The driver's hook starts the pump (refused while a teardown is in progress) and waits, bounded, on the new EventPump.StreamRequested signal - completed once the stream loop's first MoveNext has issued the StreamEvents RPC, or when the loop ends - so the advise never precedes the stream request. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
@@ -540,6 +540,9 @@ public sealed class GalaxyDriver
|
||||
// a cold gateway cache's first-load wait — longer than Galaxy's Tier-A Initialize budget. It
|
||||
// runs in the background: InitializeAsync neither waits on it nor fails because of it.
|
||||
await PrepareProbeSeedAsync(cancellationToken).ConfigureAwait(false);
|
||||
// #865: the seed's probe advises start the event pump themselves (BuildProbeWatcher's
|
||||
// pre-subscribe hook) — a driver with no authored tags never Reads/Subscribes, and the pump
|
||||
// used to start only there, so every initial ScanState value went to no stream.
|
||||
RequestProbeSeed("initialize");
|
||||
_logger.LogInformation(
|
||||
"GalaxyDriver {InstanceId} initialized — endpoint={Endpoint} clientName={ClientName}",
|
||||
@@ -730,9 +733,12 @@ public sealed class GalaxyDriver
|
||||
// The watcher publishes its probe handles into the driver's registry (reserved id) so the
|
||||
// EventPump's fan-out reaches OnPumpDataChange for ScanState events — without that every
|
||||
// probe event was dropped before it got anywhere near the watcher.
|
||||
// #865: the watcher also starts the pump before every probe SubscribeBulk — the initial ScanState
|
||||
// value only reaches a stream that is already attached (the gateway does not replay a fresh one).
|
||||
private PerPlatformProbeWatcher BuildProbeWatcher(IGalaxySubscriber subscriber) =>
|
||||
new(subscriber, _hostStatuses, _subscriptions, _logger,
|
||||
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs);
|
||||
bufferedUpdateIntervalMs: _options.MxAccess.PublishingIntervalMs,
|
||||
ensureEventStream: EnsureProbeEventStreamAsync);
|
||||
|
||||
/// <summary>
|
||||
/// Build the production gw client + session + per-capability runtime components
|
||||
@@ -2101,6 +2107,47 @@ public sealed class GalaxyDriver
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="EnsureEventPumpStarted"/> for infrastructure callers (the probe watcher) that
|
||||
/// must not fail because the runtime is going away: false — and no pump — while a teardown is in
|
||||
/// progress or there is no subscriber.
|
||||
/// </summary>
|
||||
/// <returns>The running (or just started) pump; null when none may be started.</returns>
|
||||
private EventPump? TryEnsureEventPumpStarted()
|
||||
{
|
||||
lock (_pumpLock)
|
||||
{
|
||||
if (_runtimeStopping || _disposed || _subscriber is null) return null;
|
||||
return EnsureEventPumpStarted(); // re-enters _pumpLock (Lock is reentrant); cannot throw here
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Upper bound on how long a probe subscribe waits for the pump's StreamEvents call to be issued.</summary>
|
||||
internal TimeSpan ProbeStreamRequestWait { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// #865 — the probe watcher's pre-subscribe hook: start the pump (if this runtime may) and wait,
|
||||
/// bounded, until its StreamEvents call has been issued, so the gateway has a subscriber to push
|
||||
/// the probe's initial ScanState value to. A fresh stream (after_worker_sequence = 0) is never
|
||||
/// replayed, so an initial value pushed before it attaches is lost — and ScanState rarely changes
|
||||
/// again. On timeout the subscribe proceeds anyway (logged): a late stream is no worse than before.
|
||||
/// </summary>
|
||||
private async Task EnsureProbeEventStreamAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var pump = TryEnsureEventPumpStarted();
|
||||
if (pump is null) return;
|
||||
try
|
||||
{
|
||||
await pump.StreamRequested.WaitAsync(ProbeStreamRequestWait, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"GalaxyDriver {InstanceId} event stream was not requested within {Wait} — subscribing the platform probes anyway; their initial ScanState may be missed.",
|
||||
_driverInstanceId, ProbeStreamRequestWait);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replay-gap callback for the <see cref="EventPump"/>. The gateway told us it evicted
|
||||
/// events we can never receive, so every cached value may be stale by an unknown amount.
|
||||
|
||||
@@ -44,6 +44,11 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
// driven directly (unit tests of the decoding table).
|
||||
private readonly SubscriptionRegistry? _registry;
|
||||
|
||||
// #865: invoked before every probe SubscribeBulk so the driver's event stream is attached when the
|
||||
// gateway pushes the initial ScanState value (a fresh StreamEvents does not replay earlier events,
|
||||
// and ScanState rarely changes — a lost initial value leaves the host missing indefinitely).
|
||||
private readonly Func<CancellationToken, Task>? _ensureEventStream;
|
||||
|
||||
// Tracked platform → gw item handle. Item handle 0 means the gw rejected the subscribe;
|
||||
// we keep the entry so SyncPlatformsAsync doesn't try to subscribe it again on every call.
|
||||
// Each handle carries the registry session that issued it (#859 I-A): item handles are per gateway
|
||||
@@ -79,16 +84,23 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
/// <param name="registry">The driver's subscription registry; null to keep the handles private.</param>
|
||||
/// <param name="logger">Optional logger for diagnostic messages.</param>
|
||||
/// <param name="bufferedUpdateIntervalMs">Buffered update interval in milliseconds; must be >= 0.</param>
|
||||
/// <param name="ensureEventStream">
|
||||
/// Optional: awaited before every probe SubscribeBulk to make sure the event stream that carries
|
||||
/// the initial ScanState value is running and requested (the driver starts its
|
||||
/// <see cref="EventPump"/> and waits for its StreamEvents call to be issued).
|
||||
/// </param>
|
||||
internal PerPlatformProbeWatcher(
|
||||
IGalaxySubscriber subscriber,
|
||||
HostStatusAggregator aggregator,
|
||||
SubscriptionRegistry? registry,
|
||||
ILogger? logger = null,
|
||||
int bufferedUpdateIntervalMs = 0)
|
||||
int bufferedUpdateIntervalMs = 0,
|
||||
Func<CancellationToken, Task>? ensureEventStream = null)
|
||||
{
|
||||
_subscriber = subscriber ?? throw new ArgumentNullException(nameof(subscriber));
|
||||
_aggregator = aggregator ?? throw new ArgumentNullException(nameof(aggregator));
|
||||
_registry = registry;
|
||||
_ensureEventStream = ensureEventStream;
|
||||
_logger = logger ?? NullLogger.Instance;
|
||||
if (bufferedUpdateIntervalMs < 0)
|
||||
{
|
||||
@@ -196,6 +208,10 @@ public sealed class PerPlatformProbeWatcher : IDisposable
|
||||
{
|
||||
var probeAddresses = platforms.Select(p => p + ProbeSuffix).ToArray();
|
||||
|
||||
// #865: the stream must be live BEFORE the advise — the initial value is the probe's only
|
||||
// event until the platform's scan state changes, and an unattached stream never sees it.
|
||||
if (_ensureEventStream is not null) await _ensureEventStream(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
// #859: the initial ScanState value can overtake the SubscribeBulk reply; park it until
|
||||
// PublishBindingsLocked registers the handles below (window closes at method exit).
|
||||
using var earlyEventWindow = _registry?.BeginPendingSubscribe();
|
||||
|
||||
@@ -148,6 +148,17 @@ internal sealed class EventPump : IAsyncDisposable
|
||||
|
||||
private readonly int _session;
|
||||
|
||||
private readonly TaskCompletionSource _streamRequested = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>
|
||||
/// Completes once the stream loop has issued its <c>StreamEvents</c> call (the first
|
||||
/// <c>MoveNextAsync</c> runs the subscriber's iterator synchronously up to the RPC) — or once the
|
||||
/// loop has ended without getting that far, so a waiter never hangs on a dead pump. #865: the
|
||||
/// gateway does not replay a fresh stream, so a caller whose only expected event is the initial
|
||||
/// value of an advise it is about to make waits on this first.
|
||||
/// </summary>
|
||||
public Task StreamRequested => _streamRequested.Task;
|
||||
|
||||
/// <summary>
|
||||
/// Marker written into the channel to wake the dispatch loop after a registration promoted
|
||||
/// parked early events (#859). Never dispatched itself; compared by reference.
|
||||
@@ -179,10 +190,17 @@ internal sealed class EventPump : IAsyncDisposable
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
IAsyncEnumerator<MxEvent>? enumerator = null;
|
||||
try
|
||||
{
|
||||
await foreach (var ev in _subscriber.StreamEventsAsync(ct).WithCancellation(ct).ConfigureAwait(false))
|
||||
enumerator = _subscriber.StreamEventsAsync(ct).GetAsyncEnumerator(ct);
|
||||
// Start the first MoveNext BEFORE awaiting it: that synchronous step issues the RPC, which is
|
||||
// the moment StreamRequested reports (#865).
|
||||
var next = enumerator.MoveNextAsync();
|
||||
_streamRequested.TrySetResult();
|
||||
for (; await next.ConfigureAwait(false); next = enumerator.MoveNextAsync())
|
||||
{
|
||||
var ev = enumerator.Current;
|
||||
if (ct.IsCancellationRequested) break;
|
||||
EventsReceived.Add(1, _clientTag);
|
||||
|
||||
@@ -241,6 +259,20 @@ internal sealed class EventPump : IAsyncDisposable
|
||||
}
|
||||
finally
|
||||
{
|
||||
// A loop that died before issuing the RPC must not strand a StreamRequested waiter.
|
||||
_streamRequested.TrySetResult();
|
||||
if (enumerator is not null)
|
||||
{
|
||||
try { await enumerator.DisposeAsync().ConfigureAwait(false); }
|
||||
catch (Exception ex) when (ct.IsCancellationRequested || ex is OperationCanceledException)
|
||||
{
|
||||
// Disposing a cancelled stream may surface its cancellation — shutdown noise.
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Galaxy EventPump stream enumerator dispose failed.");
|
||||
}
|
||||
}
|
||||
// Tell the dispatch loop the producer is done so it drains and exits.
|
||||
_channel.Writer.TryComplete();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user