4fdc3708f4
Under heavy parallel load (14 concurrent test processes + 14 CPU burners, with DOTNET_ThreadPool_ForceMinWorkerThreads=1 as the reproducer) the Galaxy tests with 1-3 s fixed waits failed 16/70 runs. Every failure was a background loop started with Task.Run (alarm feed, DeployWatcher, EventPump, ReconnectSupervisor) not being SCHEDULED within the ceiling - e.g. DeployWatcher's stream had not even been opened after 2 s. No production race: the code under test was checked for each (alarm feed dispatch is synchronous in-order, DeployWatcher/ReconnectSupervisor state is lock- or loop-local, subscribe registers before returning). - New shared TestWait (UntilAsync / ForAsync / HealthyAsync): 30 s presence ceiling, failure reports elapsed ms + an observed-state snapshot (#856 shape; GalaxyDriverFixRound1Tests.WaitUntilAsync now delegates to it). - Converted the short waits in GatewayGalaxyAlarmFeedTests, DeployWatcherTests, GalaxyDriverSubscribeTests, GalaxyDriverRawPathKeyingTests (its WaitForAsync never threw on timeout), EventPumpStreamFaultTests, EventPumpReplayGapTests, ReconnectSupervisorTests; FixRound2 InitBudget 2 s -> 10 s (the parked fetch never completes, so any finite bound keeps the verdict). - Absence by short sleep replaced with positive controls: DeployWatcher bootstrap and same-deploy-time (follow with a real change, assert exactly that one raise - verified red when suppression is removed), subscribe failed-tag / unsubscribed handle (follow with a live handle's event), faulted pump (wait for its onStreamFault instead of 100 ms). - Fixed sleeps that raced scheduling: subscribe-once read tests wait for the read's registration (TrackedSubscriptionCountForTests) instead of 50 ms; supervisor Dispose test waits for reopen entry instead of 50 ms; the already-Healthy WaitForHealthy test asserts synchronous completion instead of a wall-clock bound. FakeSubscriber.Map is now a ConcurrentDictionary. Loaded run (14 parallel x 5, 14 burners, ForceMinWorkerThreads=1): before 16/70 failed runs; after 0/350 (70 + 210 + 70). Without the thread-pool override: 0/70 before and after on this host. Serial: 472 passed, 8 skipped. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
108 lines
5.2 KiB
C#
108 lines
5.2 KiB
C#
using System.Diagnostics;
|
||
using Shouldly;
|
||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests;
|
||
|
||
/// <summary>
|
||
/// Shared presence waits for the Galaxy driver tests (#856, #858).
|
||
/// </summary>
|
||
/// <remarks>
|
||
/// <para>
|
||
/// Every wait here is a PRESENCE wait: it returns the instant the awaited thing happens, so a
|
||
/// generous ceiling costs nothing on a passing run. Under heavy parallel load (#858: 14 concurrent
|
||
/// test processes + 14 CPU burners, or a starved thread pool) a background loop started with
|
||
/// <c>Task.Run</c> can take seconds just to be scheduled — the old 1–3 s ceilings failed there while
|
||
/// the code under test was correct.
|
||
/// </para>
|
||
/// <para>
|
||
/// On timeout the failure reports the elapsed time and, when given, a <c>describe</c> snapshot of the
|
||
/// state actually observed, so a flake says which link of the chain never happened.
|
||
/// </para>
|
||
/// <para>
|
||
/// Never assert ABSENCE with a short sleep: follow the event that must not fire with one that MUST
|
||
/// arrive (a positive control), wait for that, then assert the absence.
|
||
/// </para>
|
||
/// </remarks>
|
||
internal static class TestWait
|
||
{
|
||
/// <summary>Ceiling for a presence wait. Only a genuinely broken run ever waits this long.</summary>
|
||
internal static readonly TimeSpan Presence = TimeSpan.FromSeconds(30);
|
||
|
||
/// <summary>Polls <paramref name="condition"/> until it holds or <see cref="Presence"/> elapses.</summary>
|
||
/// <param name="condition">The condition to wait for.</param>
|
||
/// <param name="because">What the condition means; leads the failure message.</param>
|
||
/// <param name="describe">Optional snapshot of the observed state, appended on timeout.</param>
|
||
/// <param name="timeout">Optional ceiling overriding <see cref="Presence"/>.</param>
|
||
internal static async Task UntilAsync(
|
||
Func<bool> condition, string because, Func<string>? describe = null, TimeSpan? timeout = null)
|
||
{
|
||
var limit = timeout ?? Presence;
|
||
var sw = Stopwatch.StartNew();
|
||
while (!condition())
|
||
{
|
||
if (sw.Elapsed > limit)
|
||
{
|
||
throw new ShouldAssertException(Message(because, sw, describe));
|
||
}
|
||
await Task.Delay(10).ConfigureAwait(false);
|
||
}
|
||
}
|
||
|
||
/// <summary>Waits for <paramref name="task"/> to complete, or fails after <see cref="Presence"/>.</summary>
|
||
/// <param name="task">The task that must complete (e.g. a <c>TaskCompletionSource.Task</c>).</param>
|
||
/// <param name="because">What completion means; leads the failure message.</param>
|
||
/// <param name="describe">Optional snapshot of the observed state, appended on timeout.</param>
|
||
/// <returns>A task that completes when <paramref name="task"/> does (its fault/cancellation propagates).</returns>
|
||
internal static async Task ForAsync(Task task, string because, Func<string>? describe = null)
|
||
{
|
||
var sw = Stopwatch.StartNew();
|
||
var winner = await Task.WhenAny(task, Task.Delay(Presence)).ConfigureAwait(false);
|
||
if (winner != task)
|
||
{
|
||
throw new ShouldAssertException(Message(because, sw, describe));
|
||
}
|
||
await task.ConfigureAwait(false);
|
||
}
|
||
|
||
/// <summary>Waits for <paramref name="task"/> and returns its result, or fails after <see cref="Presence"/>.</summary>
|
||
/// <typeparam name="T">The result type.</typeparam>
|
||
/// <param name="task">The task that must complete.</param>
|
||
/// <param name="because">What completion means; leads the failure message.</param>
|
||
/// <param name="describe">Optional snapshot of the observed state, appended on timeout.</param>
|
||
/// <returns>The task's result.</returns>
|
||
internal static async Task<T> ForAsync<T>(Task<T> task, string because, Func<string>? describe = null)
|
||
{
|
||
await ForAsync((Task)task, because, describe).ConfigureAwait(false);
|
||
return await task.ConfigureAwait(false);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Drives the production <see cref="ReconnectSupervisor.WaitForHealthyAsync"/> under the presence
|
||
/// ceiling; a timeout fails with the supervisor's observed state rather than a bare
|
||
/// <see cref="TaskCanceledException"/>.
|
||
/// </summary>
|
||
/// <param name="supervisor">The supervisor that must return to Healthy.</param>
|
||
/// <param name="because">What recovery means here; leads the failure message.</param>
|
||
internal static async Task HealthyAsync(ReconnectSupervisor supervisor, string because)
|
||
{
|
||
var sw = Stopwatch.StartNew();
|
||
using var cts = new CancellationTokenSource(Presence);
|
||
try
|
||
{
|
||
await supervisor.WaitForHealthyAsync(cts.Token).ConfigureAwait(false);
|
||
}
|
||
catch (OperationCanceledException) when (cts.IsCancellationRequested)
|
||
{
|
||
throw new ShouldAssertException(Message(because, sw,
|
||
() => $"state={supervisor.CurrentState} lastError={supervisor.LastError ?? "<none>"}"));
|
||
}
|
||
}
|
||
|
||
private static string Message(string because, Stopwatch sw, Func<string>? describe)
|
||
{
|
||
var observed = describe is null ? "" : $" Observed: {describe()}";
|
||
return $"{because} (not met after {sw.ElapsedMilliseconds} ms).{observed}";
|
||
}
|
||
}
|