Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests/MTConnectStaleCursorTests.cs
T
Joseph Doherty f48a065c21 fix(mtconnect): a superseded pump writes no health, flags or index state (review I-1, m2, m3)
- The pump's delivered/degraded writes (stream flag, live flag, PublishHealthy /
  Degrade) now happen in TryMarkStreamDelivered / TryDegradeStream: one critical
  section under _subscriptionLock that re-checks the session generation. A pump
  wedged in a subscriber past its last check can no longer report a replaced
  session's (down) stream Healthy and make the read path trust it.
- m3: the index and condition tracker are installed in the same lock as the
  session + generation and captured under it (TryCaptureSessionState), so a stale
  chunk cannot fold into the new session's index.
- Teardown also bumps the generation, so an abandoned pump cannot write over the
  shutdown's health either.
- m2: an Agent restart clears the live flag before clearing the index, so reads
  go to /current until the re-baseline lands.
Red test: the wedged-pump harness with the new stream failing, plus a positive
control (the current pump's chunk restores Healthy).

Refs #743

Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT
2026-10-22 16:27:23 -04:00

186 lines
8.0 KiB
C#

using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
/// <summary>
/// Review 2026-10-21 R06:T-1 — a pump the lifecycle abandoned (its teardown wait timed out) that finished after
/// the restart installed the NEW session published its stale cursor onto it: the old Agent's instanceId and
/// sequence replaced the new session's, so the next pump to start treated its first chunk as an Agent restart
/// (index cleared, a Bad flicker on every subscribed tag, a needless re-baseline). The pump now carries the
/// session generation it was started against and its cursor publish is dropped once that generation is gone.
/// </summary>
public sealed class MTConnectStaleCursorTests
{
private const long InstanceB = 1655000099L;
private const long NextSequenceB = 500L;
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
/// <summary>The abandoned pump's late cursor publish leaves the new session's cursor alone.</summary>
[Fact]
public async Task An_abandoned_pump_does_not_publish_its_cursor_onto_the_new_session()
{
var clientA = CannedAgentClient.FromFixtures();
var clientB = CannedAgentClient.FromFixtures();
clientB.Current = new MTConnectStreamsResult(InstanceB, NextSequenceB, NextSequenceB - 10, clientB.Current.Observations);
var clients = new Queue<CannedAgentClient>([clientA, clientB]);
var driver = new MTConnectDriver(
new MTConnectDriverOptions
{
AgentUri = "http://fixture-agent:5000",
RequestTimeoutMs = 5000,
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
},
"mt-t1",
_ => clients.Dequeue());
await driver.InitializeAsync("{}", Ct);
var wedgeOnce = 0;
var wedge = new ManualResetEventSlim(false);
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
driver.OnDataChange += (_, e) =>
{
if (e.Snapshot.Value is not 1.5d || Interlocked.Exchange(ref wedgeOnce, 1) != 0) return;
entered.TrySetResult();
wedge.Wait();
};
try
{
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
var pumpA = driver.SampleStreamTask.ShouldNotBeNull();
_ = clientA.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5"));
await entered.Task.WaitAsync(Watchdog, Ct);
// The restart gives up on the wedged pump (bounded teardown wait), retires clientA and installs B.
await driver.InitializeAsync("{}", Ct).WaitAsync(Watchdog, Ct);
driver.AgentInstanceId.ShouldBe(InstanceB, "positive control: the new session is installed");
pumpA.IsCompleted.ShouldBeFalse("pump A must still be alive — it is the late writer under test");
// Now let the abandoned pump finish: its finally publishes (instanceA, 113).
wedge.Set();
await pumpA.WaitAsync(Watchdog, Ct);
driver.AgentInstanceId.ShouldBe(InstanceB, "a superseded pump must not write the old Agent's instanceId");
driver.AgentNextSequence.ShouldBe(NextSequenceB, "nor its cursor");
}
finally
{
wedge.Set();
await driver.ShutdownAsync(CancellationToken.None);
}
}
/// <summary>
/// Positive control: the CURRENT pump's own cursor publish still lands — a last unsubscribe stops it, and the
/// session's cursor moves to where that pump got to.
/// </summary>
[Fact]
public async Task The_current_pump_still_publishes_its_cursor_when_it_stops()
{
var client = CannedAgentClient.FromFixtures();
var driver = new MTConnectDriver(
new MTConnectDriverOptions
{
AgentUri = "http://fixture-agent:5000",
RequestTimeoutMs = 5000,
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
},
"mt-t1-ok",
_ => client);
await driver.InitializeAsync("{}", Ct);
try
{
var handle = await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
await client.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5")).WaitAsync(Watchdog, Ct);
await driver.UnsubscribeAsync(handle, Ct);
driver.AgentNextSequence.ShouldBe(113);
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
/// <summary>
/// Review I-1: a pump wedged inside its fan-out that comes back after the restart must not restore the NEW
/// session's health (nor mark its stream live for the read path) while the new pump's stream is down. Positive
/// control: the CURRENT pump's own chunk still restores Healthy.
/// </summary>
[Fact]
public async Task An_abandoned_pump_does_not_restore_the_new_sessions_health()
{
var clientA = CannedAgentClient.FromFixtures();
var clientB = CannedAgentClient.FromFixtures();
clientB.SampleFailure = new IOException("connection refused");
var clients = new Queue<CannedAgentClient>([clientA, clientB]);
var driver = new MTConnectDriver(
new MTConnectDriverOptions
{
AgentUri = "http://fixture-agent:5000",
RequestTimeoutMs = 5000,
Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 50, MaxBackoffMs = 200 },
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
},
"mt-i1",
_ => clients.Dequeue());
await driver.InitializeAsync("{}", Ct);
var wedgeOnce = 0;
var wedge = new ManualResetEventSlim(false);
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
driver.OnDataChange += (_, e) =>
{
if (e.Snapshot.Value is not 1.5d || Interlocked.Exchange(ref wedgeOnce, 1) != 0) return;
entered.TrySetResult();
wedge.Wait();
};
try
{
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
var pumpA = driver.SampleStreamTask.ShouldNotBeNull();
_ = clientA.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "1.5"));
await entered.Task.WaitAsync(Watchdog, Ct);
await driver.InitializeAsync("{}", Ct).WaitAsync(Watchdog, Ct);
// The new session's pump fails its /sample: the driver goes Degraded.
await clientB.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
await WaitUntilAsync(() => driver.GetHealth().State == DriverState.Degraded, "the new stream is down");
wedge.Set();
await pumpA.WaitAsync(Watchdog, Ct);
driver.GetHealth().State.ShouldBe(
DriverState.Degraded, "a superseded pump must not report the new session's stream healthy");
// Positive control: the new session's own stream recovering does restore Healthy.
clientB.SampleFailure = null;
var calls = clientB.SampleCallCount;
await clientB.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
await clientB.PumpAsync(MTConnectPumpGenerationChunks.Pos(108, 113, "2.5")).WaitAsync(Watchdog, Ct);
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
}
finally
{
wedge.Set();
await driver.ShutdownAsync(CancellationToken.None);
}
}
private static async Task WaitUntilAsync(Func<bool> condition, string because)
{
var deadline = DateTime.UtcNow + Watchdog;
while (!condition() && DateTime.UtcNow < deadline) await Task.Delay(20, Ct);
condition().ShouldBeTrue(because);
}
}