f48a065c21
- The pump's delivered/degraded writes (stream flag, live flag, PublishHealthy / Degrade) now happen in TryMarkStreamDelivered / TryDegradeStream: one critical section under _subscriptionLock that re-checks the session generation. A pump wedged in a subscriber past its last check can no longer report a replaced session's (down) stream Healthy and make the read path trust it. - m3: the index and condition tracker are installed in the same lock as the session + generation and captured under it (TryCaptureSessionState), so a stale chunk cannot fold into the new session's index. - Teardown also bumps the generation, so an abandoned pump cannot write over the shutdown's health either. - m2: an Agent restart clears the live flag before clearing the index, so reads go to /current until the re-baseline lands. Red test: the wedged-pump harness with the new stream failing, plus a positive control (the current pump's chunk restores Healthy). Refs #743 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
186 lines
8.0 KiB
C#
186 lines
8.0 KiB
C#
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
|
|
|
|
/// <summary>
|
|
/// Review 2026-10-21 R06:T-1 — a pump the lifecycle abandoned (its teardown wait timed out) that finished after
|
|
/// the restart installed the NEW session published its stale cursor onto it: the old Agent's instanceId and
|
|
/// sequence replaced the new session's, so the next pump to start treated its first chunk as an Agent restart
|
|
/// (index cleared, a Bad flicker on every subscribed tag, a needless re-baseline). The pump now carries the
|
|
/// session generation it was started against and its cursor publish is dropped once that generation is gone.
|
|
/// </summary>
|
|
public sealed class MTConnectStaleCursorTests
|
|
{
|
|
private const long InstanceB = 1655000099L;
|
|
private const long NextSequenceB = 500L;
|
|
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);
|
|
|
|
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
|
|
|
/// <summary>The abandoned pump's late cursor publish leaves the new session's cursor alone.</summary>
|
|
[Fact]
|
|
public async Task An_abandoned_pump_does_not_publish_its_cursor_onto_the_new_session()
|
|
{
|
|
var clientA = CannedAgentClient.FromFixtures();
|
|
var clientB = CannedAgentClient.FromFixtures();
|
|
clientB.Current = new MTConnectStreamsResult(InstanceB, NextSequenceB, NextSequenceB - 10, clientB.Current.Observations);
|
|
var clients = new Queue<CannedAgentClient>([clientA, clientB]);
|
|
|
|
var driver = new MTConnectDriver(
|
|
new MTConnectDriverOptions
|
|
{
|
|
AgentUri = "http://fixture-agent:5000",
|
|
RequestTimeoutMs = 5000,
|
|
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
|
|
},
|
|
"mt-t1",
|
|
_ => clients.Dequeue());
|
|
await driver.InitializeAsync("{}", Ct);
|
|
|
|
var wedgeOnce = 0;
|
|
var wedge = new ManualResetEventSlim(false);
|
|
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
driver.OnDataChange += (_, e) =>
|
|
{
|
|
if (e.Snapshot.Value is not 1.5d || Interlocked.Exchange(ref wedgeOnce, 1) != 0) return;
|
|
entered.TrySetResult();
|
|
wedge.Wait();
|
|
};
|
|
|
|
try
|
|
{
|
|
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
|
|
var pumpA = driver.SampleStreamTask.ShouldNotBeNull();
|
|
_ = clientA.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5"));
|
|
await entered.Task.WaitAsync(Watchdog, Ct);
|
|
|
|
// The restart gives up on the wedged pump (bounded teardown wait), retires clientA and installs B.
|
|
await driver.InitializeAsync("{}", Ct).WaitAsync(Watchdog, Ct);
|
|
driver.AgentInstanceId.ShouldBe(InstanceB, "positive control: the new session is installed");
|
|
pumpA.IsCompleted.ShouldBeFalse("pump A must still be alive — it is the late writer under test");
|
|
|
|
// Now let the abandoned pump finish: its finally publishes (instanceA, 113).
|
|
wedge.Set();
|
|
await pumpA.WaitAsync(Watchdog, Ct);
|
|
|
|
driver.AgentInstanceId.ShouldBe(InstanceB, "a superseded pump must not write the old Agent's instanceId");
|
|
driver.AgentNextSequence.ShouldBe(NextSequenceB, "nor its cursor");
|
|
}
|
|
finally
|
|
{
|
|
wedge.Set();
|
|
await driver.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Positive control: the CURRENT pump's own cursor publish still lands — a last unsubscribe stops it, and the
|
|
/// session's cursor moves to where that pump got to.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task The_current_pump_still_publishes_its_cursor_when_it_stops()
|
|
{
|
|
var client = CannedAgentClient.FromFixtures();
|
|
var driver = new MTConnectDriver(
|
|
new MTConnectDriverOptions
|
|
{
|
|
AgentUri = "http://fixture-agent:5000",
|
|
RequestTimeoutMs = 5000,
|
|
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
|
|
},
|
|
"mt-t1-ok",
|
|
_ => client);
|
|
await driver.InitializeAsync("{}", Ct);
|
|
try
|
|
{
|
|
var handle = await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
|
|
await client.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5")).WaitAsync(Watchdog, Ct);
|
|
|
|
await driver.UnsubscribeAsync(handle, Ct);
|
|
|
|
driver.AgentNextSequence.ShouldBe(113);
|
|
}
|
|
finally
|
|
{
|
|
await driver.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Review I-1: a pump wedged inside its fan-out that comes back after the restart must not restore the NEW
|
|
/// session's health (nor mark its stream live for the read path) while the new pump's stream is down. Positive
|
|
/// control: the CURRENT pump's own chunk still restores Healthy.
|
|
/// </summary>
|
|
[Fact]
|
|
public async Task An_abandoned_pump_does_not_restore_the_new_sessions_health()
|
|
{
|
|
var clientA = CannedAgentClient.FromFixtures();
|
|
var clientB = CannedAgentClient.FromFixtures();
|
|
clientB.SampleFailure = new IOException("connection refused");
|
|
var clients = new Queue<CannedAgentClient>([clientA, clientB]);
|
|
|
|
var driver = new MTConnectDriver(
|
|
new MTConnectDriverOptions
|
|
{
|
|
AgentUri = "http://fixture-agent:5000",
|
|
RequestTimeoutMs = 5000,
|
|
Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 50, MaxBackoffMs = 200 },
|
|
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
|
|
},
|
|
"mt-i1",
|
|
_ => clients.Dequeue());
|
|
await driver.InitializeAsync("{}", Ct);
|
|
|
|
var wedgeOnce = 0;
|
|
var wedge = new ManualResetEventSlim(false);
|
|
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
driver.OnDataChange += (_, e) =>
|
|
{
|
|
if (e.Snapshot.Value is not 1.5d || Interlocked.Exchange(ref wedgeOnce, 1) != 0) return;
|
|
entered.TrySetResult();
|
|
wedge.Wait();
|
|
};
|
|
|
|
try
|
|
{
|
|
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
|
|
var pumpA = driver.SampleStreamTask.ShouldNotBeNull();
|
|
_ = clientA.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5"));
|
|
await entered.Task.WaitAsync(Watchdog, Ct);
|
|
|
|
await driver.InitializeAsync("{}", Ct).WaitAsync(Watchdog, Ct);
|
|
|
|
// The new session's pump fails its /sample: the driver goes Degraded.
|
|
await clientB.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
|
|
await WaitUntilAsync(() => driver.GetHealth().State == DriverState.Degraded, "the new stream is down");
|
|
|
|
wedge.Set();
|
|
await pumpA.WaitAsync(Watchdog, Ct);
|
|
|
|
driver.GetHealth().State.ShouldBe(
|
|
DriverState.Degraded, "a superseded pump must not report the new session's stream healthy");
|
|
|
|
// Positive control: the new session's own stream recovering does restore Healthy.
|
|
clientB.SampleFailure = null;
|
|
var calls = clientB.SampleCallCount;
|
|
await clientB.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
|
|
await clientB.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "2.5")).WaitAsync(Watchdog, Ct);
|
|
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
|
}
|
|
finally
|
|
{
|
|
wedge.Set();
|
|
await driver.ShutdownAsync(CancellationToken.None);
|
|
}
|
|
}
|
|
|
|
private static async Task WaitUntilAsync(Func<bool> condition, string because)
|
|
{
|
|
var deadline = DateTime.UtcNow + Watchdog;
|
|
while (!condition() && DateTime.UtcNow < deadline) await Task.Delay(20, Ct);
|
|
condition().ShouldBeTrue(because);
|
|
}
|
|
}
|