1c1705fa77
- StreamRequestWait 1 s (under Tier-A Read 2 s / Subscribe 5 s budgets) and a stale pump is only waited on while the supervisor is reopening or replaying (or a restart is underway); in a reconnect backoff the advise proceeds at once. The no-stream log is a rate-limited Warning while Healthy, Debug otherwise. - A current-session pump whose loop has ended is not a stream: if the supervisor already reports Healthy (its fault was swallowed while it was recovering) the failure is re-reported to start a new cycle. - Subscribe/Read no longer start the pump directly (only AwaitCurrentEventStreamAsync does, honouring the replay restart flag); the teardown refusal is an explicit ThrowIfRuntimeStopping. - SubscriptionRegistry.ResolveOrPark fast path: the handle lookup and the open-window check were not atomic, so a subscribe that registered the handle and closed its window in between got its initial value dropped as stale (found by a stress run: the first event of a bulk advise lost). It re-looks-up before declaring stale. Deterministic test via a seam. - Tests: first Read on a fresh driver, replay of a data subscription. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
341 lines
18 KiB
C#
341 lines
18 KiB
C#
using System.Runtime.CompilerServices;
|
|
using System.Threading.Channels;
|
|
using Google.Protobuf.WellKnownTypes;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Browse;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
|
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
|
|
|
/// <summary>
|
|
/// #865: the platform ScanState probes' initial value. The gateway pushes an advise's current value
|
|
/// once, as its first event, and a FRESH <c>StreamEvents</c> (<c>after_worker_sequence = 0</c>) never
|
|
/// replays — so a probe advised before the driver's event stream is attached never gets a value, and
|
|
/// ScanState rarely changes again. The event pump used to start only on the first Read/Subscribe,
|
|
/// so a driver with no authored tags (or any driver before its first read) had no host statuses.
|
|
/// </summary>
|
|
public sealed class ProbeInitialValueTests
|
|
{
|
|
private static readonly string[] Platforms = ["CTRX_DEV", "DevWinPlatform", "DevAppEngine"];
|
|
|
|
private static GalaxyDriverOptions Opts() => new(
|
|
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
|
new GalaxyMxAccessOptions("ProbeInitialValue"),
|
|
new GalaxyRepositoryOptions(WatchDeployEvents: false),
|
|
new GalaxyReconnectOptions());
|
|
|
|
private sealed class FixedPlatformSource(params string[] platforms) : IGalaxyPlatformSource
|
|
{
|
|
public Task<IReadOnlyList<string>> GetPlatformTagNamesAsync(CancellationToken cancellationToken)
|
|
=> Task.FromResult<IReadOnlyList<string>>(platforms);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Models the gateway's event semantics: every advise pushes its current value exactly once, and
|
|
/// only to a stream that is attached at that moment (no replay for a fresh stream). Records the
|
|
/// order in which the stream was requested and the advises were made.
|
|
/// </summary>
|
|
private sealed class GatewayLikeSubscriber : IGalaxySubscriber
|
|
{
|
|
private readonly Lock _gate = new();
|
|
private Channel<MxEvent>? _attached;
|
|
private int _nextHandle;
|
|
|
|
/// <summary>"stream" per StreamEvents call, "subscribe:<refs>" per SubscribeBulk, in call order.</summary>
|
|
public List<string> Calls { get; } = [];
|
|
|
|
/// <summary>Initial values pushed while no stream was attached — lost, as on the real gateway.</summary>
|
|
public int DroppedInitialValues;
|
|
|
|
/// <summary>The ScanState value the next advise reports.</summary>
|
|
public volatile bool ScanStateRunning = true;
|
|
|
|
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
|
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
|
{
|
|
var results = new List<SubscribeResult>();
|
|
lock (_gate)
|
|
{
|
|
Calls.Add("subscribe:" + string.Join(",", fullReferences));
|
|
foreach (var r in fullReferences)
|
|
{
|
|
var h = ++_nextHandle;
|
|
results.Add(new SubscribeResult { TagAddress = r, ItemHandle = h, WasSuccessful = true });
|
|
var initial = new MxEvent
|
|
{
|
|
Family = MxEventFamily.OnDataChange,
|
|
ItemHandle = h,
|
|
Value = new MxValue { BoolValue = ScanStateRunning },
|
|
Quality = 192,
|
|
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
|
};
|
|
if (_attached is null || !_attached.Writer.TryWrite(initial)) Interlocked.Increment(ref DroppedInitialValues);
|
|
}
|
|
}
|
|
return Task.FromResult<IReadOnlyList<SubscribeResult>>(results);
|
|
}
|
|
|
|
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken) => Task.CompletedTask;
|
|
|
|
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
|
{
|
|
// Attached synchronously on the first MoveNext — the point a real gRPC call is issued.
|
|
var mine = Channel.CreateUnbounded<MxEvent>();
|
|
lock (_gate)
|
|
{
|
|
Calls.Add("stream");
|
|
_attached = mine;
|
|
}
|
|
try
|
|
{
|
|
await foreach (var ev in mine.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
yield return ev;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (ReferenceEquals(_attached, mine)) _attached = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
public string[] CallsSnapshot()
|
|
{
|
|
lock (_gate) return [.. Calls];
|
|
}
|
|
}
|
|
|
|
private static HostState? StateOf(GalaxyDriver driver, string host) =>
|
|
driver.GetHostStatuses().FirstOrDefault(s => s.HostName == host)?.State;
|
|
|
|
private static string Describe(GalaxyDriver driver, GatewayLikeSubscriber sub) =>
|
|
$"hosts=[{string.Join(", ", driver.GetHostStatuses().Select(s => $"{s.HostName}={s.State}"))}] " +
|
|
$"calls=[{string.Join(" | ", sub.CallsSnapshot())}] dropped={sub.DroppedInitialValues} pump={driver.HasEventPumpForTests}";
|
|
|
|
/// <summary>
|
|
/// A driver with NO authored tags — nothing ever Reads or Subscribes — still reports every seeded
|
|
/// platform with its initial ScanState. Before #865 the pump never started, every initial value
|
|
/// was pushed to no stream, and only the transport entry ever appeared.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Driver_with_no_authored_tags_reports_every_platform_from_its_initial_ScanState()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
|
platformSource: new FixedPlatformSource(Platforms));
|
|
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
|
|
await TestWait.UntilAsync(
|
|
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
|
"every seeded platform must surface Running from its initial ScanState, with no Read/Subscribe",
|
|
() => Describe(driver, sub));
|
|
sub.DroppedInitialValues.ShouldBe(0, "no probe's initial value may be pushed before the stream is attached");
|
|
driver.TrackedSubscriptionCountForTests.ShouldBe(1, "only the probe entry — no data subscription was ever made");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Ordering: the event stream is requested BEFORE the first probe SubscribeBulk. On the old code
|
|
/// the probe advise went out with no stream at all.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Probe_subscribe_is_issued_only_after_the_event_stream_is_requested()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
|
platformSource: new FixedPlatformSource(Platforms));
|
|
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
await TestWait.UntilAsync(
|
|
() => driver.ProbeWatcherForTests?.WatchedPlatforms.Count == Platforms.Length,
|
|
"positive control: the background seed must advise every platform",
|
|
() => Describe(driver, sub));
|
|
|
|
var calls = sub.CallsSnapshot();
|
|
var firstSubscribe = Array.FindIndex(calls, c => c.StartsWith("subscribe:", StringComparison.Ordinal));
|
|
var firstStream = Array.IndexOf(calls, "stream");
|
|
firstSubscribe.ShouldBeGreaterThanOrEqualTo(0);
|
|
firstStream.ShouldBeGreaterThanOrEqualTo(0, $"the probe seed must start the event stream; calls=[{string.Join(" | ", calls)}]");
|
|
firstStream.ShouldBeLessThan(firstSubscribe, $"the stream must be requested before the probe advise; calls=[{string.Join(" | ", calls)}]");
|
|
}
|
|
|
|
/// <summary>
|
|
/// A reconnect replay re-advises the probes on the recreated session; their new initial values must
|
|
/// arrive too (the replay restarts the pump first). The value flips to Stopped so the presence wait
|
|
/// can only pass on a post-replay event.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Reconnect_replay_delivers_the_re_advised_probes_initial_values()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
|
platformSource: new FixedPlatformSource(Platforms));
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
await TestWait.UntilAsync(
|
|
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
|
"positive control: the initial seed reports Running",
|
|
() => Describe(driver, sub));
|
|
|
|
sub.ScanStateRunning = false;
|
|
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
|
|
|
await TestWait.UntilAsync(
|
|
() => Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
|
|
"the re-advised probes' initial values (Stopped) must reach the host statuses after a replay",
|
|
() => Describe(driver, sub));
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review finding: a probe advised in the reopen → replay window (the session was swapped, the old
|
|
/// pump still streams the dead session) must still get its initial value. The advise waits for the
|
|
/// replay's restarted pump instead of advising onto the stale one (its value would route under the
|
|
/// old session and be lost, and the replay skips a current-session handle) or starting a competing
|
|
/// stream (the gateway's default single-subscriber mode refuses a second attach).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Probe_advised_between_reopen_and_replay_still_gets_its_initial_value()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub,
|
|
platformSource: new FixedPlatformSource(Platforms))
|
|
{
|
|
RecreateOverrideForTests = _ => Task.CompletedTask,
|
|
};
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
await TestWait.UntilAsync(
|
|
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
|
"positive control: the initial seed reports Running",
|
|
() => Describe(driver, sub));
|
|
var oldPump = driver.EventPumpInstanceForTests;
|
|
|
|
await driver.InvokeReopenForTestAsync(CancellationToken.None); // session swapped; no replay yet
|
|
sub.ScanStateRunning = false;
|
|
var callsBeforeWindow = sub.CallsSnapshot().Length;
|
|
// The window advise must WAIT for the replay's current-session pump rather than advise onto the
|
|
// stale one (or start a competing stream).
|
|
var sync = driver.ProbeWatcherForTests.ShouldNotBeNull()
|
|
.SyncPlatformsAsync([.. Platforms, "LatePlatform"], CancellationToken.None);
|
|
|
|
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
|
await TestWait.ForAsync(sync, "the window advise completes once the replay restarted the pump");
|
|
|
|
await TestWait.UntilAsync(
|
|
() => StateOf(driver, "LatePlatform") == HostState.Stopped
|
|
&& Platforms.All(p => StateOf(driver, p) == HostState.Stopped),
|
|
"the window advise and the replay's re-advises all get their initial values",
|
|
() => Describe(driver, sub));
|
|
driver.EventPumpInstanceForTests.ShouldNotBeSameAs(oldPump, "the replay restarted the stale pump");
|
|
var windowCalls = sub.CallsSnapshot()[callsBeforeWindow..];
|
|
var lateAdvise = Array.FindIndex(windowCalls, c => c.Contains("LatePlatform", StringComparison.Ordinal));
|
|
var newStream = Array.IndexOf(windowCalls, "stream");
|
|
newStream.ShouldBeGreaterThanOrEqualTo(0, $"the replay must start a new stream; calls=[{string.Join(" | ", windowCalls)}]");
|
|
lateAdvise.ShouldBeGreaterThan(newStream, $"the window advise must follow the new stream; calls=[{string.Join(" | ", windowCalls)}]");
|
|
windowCalls.Count(c => c == "stream").ShouldBe(1, "no competing stream may be started for the window advise");
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The data paths too: a first SubscribeAsync and a first ReadAsync on a driver whose pump was never
|
|
/// started request the stream BEFORE advising, so their initial values arrive (a read of a value that
|
|
/// never changes again would otherwise time out).
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task First_subscribe_and_read_advise_only_after_the_stream_is_requested()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
|
{
|
|
ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
|
|
};
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
var delivered = new TaskCompletionSource<DataChangeEventArgs>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
driver.OnDataChange += (_, e) => delivered.TrySetResult(e);
|
|
|
|
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
|
var calls = sub.CallsSnapshot();
|
|
Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the advise; calls=[{string.Join(" | ", calls)}]");
|
|
(await TestWait.ForAsync(delivered.Task, "the subscribe's initial value must arrive", () => Describe(driver, sub)))
|
|
.FullReference.ShouldBe("Tank01.Level");
|
|
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>A first ReadAsync on a fresh driver (no pump yet) requests the stream before its advise.</summary>
|
|
[Fact]
|
|
public async Task First_read_advises_only_after_the_stream_is_requested()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub)
|
|
{
|
|
ReadViaSubscribeDeadline = TimeSpan.FromSeconds(10),
|
|
};
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
driver.HasEventPumpForTests.ShouldBeFalse("precondition: the read is the first advise");
|
|
|
|
var read = await driver.ReadAsync(["Tank02.Level"], CancellationToken.None);
|
|
|
|
var calls = sub.CallsSnapshot();
|
|
Array.IndexOf(calls, "stream").ShouldBe(0, $"the stream must be requested before the read's advise; calls=[{string.Join(" | ", calls)}]");
|
|
read[0].StatusCode.ShouldBe(0u, "the read's initial value must arrive (not BadTimeout)");
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The replay's re-advise of DATA subscriptions (which has no per-advise hook) waits for the restarted
|
|
/// pump's stream, so each re-advised tag's new initial value arrives. The value flips so only a
|
|
/// post-replay event can satisfy the wait.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Reconnect_replay_delivers_re_advised_data_subscriptions_initial_values()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
var last = new System.Collections.Concurrent.ConcurrentDictionary<string, object?>();
|
|
driver.OnDataChange += (_, e) => last[e.FullReference] = e.Snapshot.Value;
|
|
|
|
await driver.SubscribeAsync(["Tank01.Level"], TimeSpan.Zero, CancellationToken.None);
|
|
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is true,
|
|
"positive control: the subscribe's initial value arrives", () => Describe(driver, sub));
|
|
|
|
sub.ScanStateRunning = false; // the fake's next initial value
|
|
await driver.InvokeReplayForTestAsync(CancellationToken.None);
|
|
|
|
await TestWait.UntilAsync(() => last.TryGetValue("Tank01.Level", out var v) && v is false,
|
|
"the replay's re-advise must deliver its new initial value",
|
|
() => Describe(driver, sub) + $" last=[{string.Join(",", last.Select(kv => $"{kv.Key}={kv.Value}"))}] parked={driver.ParkedEarlyEventCountForTests} h2=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(2).Select(b => b.FullReference))}] h1=[{string.Join(",", driver.SubscriptionsForTests.ResolveSubscribers(1).Select(b => b.FullReference))}]");
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A probe sync on a driver whose pump was never started (no platform source, e.g. the
|
|
/// DiscoverAsync path) still starts it before advising — the watcher's pre-subscribe hook, not just
|
|
/// the init path.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task Probe_sync_without_a_platform_source_still_starts_the_stream_first()
|
|
{
|
|
var sub = new GatewayLikeSubscriber();
|
|
// No platform source: init starts no pump; the probe set arrives through SyncPlatformsAsync.
|
|
await using var driver = new GalaxyDriver("drv", Opts(), hierarchySource: null, subscriber: sub);
|
|
await driver.InitializeAsync("{}", CancellationToken.None);
|
|
driver.HasEventPumpForTests.ShouldBeFalse("precondition: nothing has started the pump yet");
|
|
|
|
await driver.ProbeWatcherForTests.ShouldNotBeNull().SyncPlatformsAsync(Platforms, CancellationToken.None);
|
|
|
|
await TestWait.UntilAsync(
|
|
() => Platforms.All(p => StateOf(driver, p) == HostState.Running),
|
|
"the watcher must start the stream before advising, so the initial values arrive",
|
|
() => Describe(driver, sub));
|
|
sub.DroppedInitialValues.ShouldBe(0);
|
|
}
|
|
}
|