using System.Collections.Concurrent;
using System.Net.Http;
using System.Net.Sockets;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
///
/// lmxopcua#846 — a REFUSED Agent connection is a definite loss: after CommsLossConfirmMs of nothing but
/// refusals every subscribed value goes BadNoCommunication (instead of Good-but-stale until the ~90 s
/// watchdog), and the first delivery afterwards republishes every value Good — including unchanged ones. Ambiguous
/// failures (timeouts, a closed stream) keep the watchdog contract. Plus the raised-count ceiling is clamped to the
/// Agent's bufferSize.
///
public sealed class MTConnectCommsLossTests
{
private const long Instance = 1655000000L;
private const long Primed = 108L;
private const uint BadNoCommunication = 0x80310000u;
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static MTConnectStreamsResult Chunk(long first, int observations, long? last = null)
{
var obs = Enumerable.Range(0, observations)
.Select(i => new MTConnectObservation(
"dev1_pos", (first + i).ToString(System.Globalization.CultureInfo.InvariantCulture),
new DateTime(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc).AddSeconds(first + i)))
.ToList();
return new MTConnectStreamsResult(Instance, first + observations, first, obs, last);
}
private static Exception Refused() =>
new HttpRequestException("Connection refused (fixture-agent:5000)", new SocketException((int)SocketError.ConnectionRefused));
private static async Task<(MTConnectDriver Driver, CannedAgentClient Client, ConcurrentQueue Seen)>
SubscribedAsync(int commsLossConfirmMs, int sampleCount = 1000, long? bufferSize = null)
{
var client = CannedAgentClient.FromFixtures();
if (bufferSize is { } size) client.Probe = client.Probe with { BufferSize = size };
var driver = new MTConnectDriver(
new MTConnectDriverOptions
{
AgentUri = "http://fixture-agent:5000",
RequestTimeoutMs = 5000,
SampleCount = sampleCount,
CommsLossConfirmMs = commsLossConfirmMs,
Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 0, MaxBackoffMs = 200 },
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
},
"mt-846",
_ => client);
var seen = new ConcurrentQueue();
driver.OnDataChange += (_, e) => seen.Enqueue(e);
await driver.InitializeAsync("{}", Ct);
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
await client.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
return (driver, client, seen);
}
private static async Task WaitUntil(Func condition, int timeoutMs = 10_000)
{
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
while (!condition())
{
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
await Task.Delay(20, Ct);
}
}
[Fact]
public void Only_a_refused_connection_is_a_definite_loss()
{
MTConnectDriver.IsDefiniteConnectionLoss(Refused()).ShouldBeTrue();
MTConnectDriver.IsDefiniteConnectionLoss(new SocketException((int)SocketError.ConnectionRefused)).ShouldBeTrue();
MTConnectDriver.IsDefiniteConnectionLoss(new AggregateException(Refused())).ShouldBeTrue();
MTConnectDriver.IsDefiniteConnectionLoss(null).ShouldBeFalse();
MTConnectDriver.IsDefiniteConnectionLoss(new TimeoutException("no answer")).ShouldBeFalse();
MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("reset", new SocketException((int)SocketError.ConnectionReset))).ShouldBeFalse();
MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("dns", new SocketException((int)SocketError.HostNotFound))).ShouldBeFalse();
MTConnectDriver.IsDefiniteConnectionLoss(new MTConnectStreamEndedException(MTConnectStreamEndReason.ConnectionClosed, "x", 3)).ShouldBeFalse();
}
[Fact]
public async Task A_refused_agent_marks_values_bad_at_once_and_the_first_delivery_republishes_them_good()
{
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0);
try
{
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
seen.Last().Snapshot.StatusCode.ShouldBe(0u);
client.SampleFailure = Refused();
client.EndStream(); // the connection drops; every reconnect is refused
await WaitUntil(() => seen.Last().Snapshot.StatusCode == BadNoCommunication);
driver.CommsLossMarked.ShouldBeTrue();
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
var badCount = seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication);
await Task.Delay(400, Ct); // more refused attempts: marked ONCE per outage
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(badCount);
// The Agent is back. An observation-free heartbeat is enough: the UNCHANGED value comes back Good.
client.SampleFailure = null;
var calls = client.SampleCallCount;
await client.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
await client.PumpAsync(Chunk(Primed + 1, 0)).WaitAsync(Watchdog, Ct);
await WaitUntil(() => seen.Last().Snapshot.StatusCode == 0u);
var recovered = seen.Last().Snapshot;
recovered.Value.ShouldBe(108.0);
driver.CommsLossMarked.ShouldBeFalse();
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
[Fact]
public async Task An_ambiguous_failure_keeps_the_watchdog_contract()
{
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0);
try
{
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
client.SampleFailure = new TimeoutException("the Agent did not answer /sample in time");
var calls = client.SampleCallCount;
client.EndStream();
await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication);
driver.CommsLossMarked.ShouldBeFalse();
driver.GetHealth().State.ShouldBe(DriverState.Degraded); // the watchdog still has its evidence
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
[Theory]
[InlineData(60_000)] // a confirmation window not yet elapsed
[InlineData(-1)] // the fast path disabled
public async Task Refusals_inside_the_confirmation_window_or_with_the_fast_path_off_mark_nothing(int confirmMs)
{
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: confirmMs);
try
{
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
client.SampleFailure = Refused();
var calls = client.SampleCallCount;
client.EndStream();
await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication);
driver.CommsLossMarked.ShouldBeFalse();
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
[Fact]
public async Task A_short_confirmation_window_marks_once_the_refusals_have_lasted_it()
{
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 300);
try
{
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
client.SampleFailure = Refused();
var started = DateTime.UtcNow;
client.EndStream();
await WaitUntil(() => driver.CommsLossMarked, timeoutMs: 5000);
(DateTime.UtcNow - started).ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(280));
seen.Last().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
// ---- the raised-count ceiling vs the Agent's bufferSize ----
[Theory]
[InlineData(1000, 0, 16_000)] // unknown buffer: the 16× rule
[InlineData(1000, 4096, 4096)] // a known buffer caps it
[InlineData(1000, 131_072, 16_000)] // a big buffer changes nothing
[InlineData(1000, 500, 1000)] // never below the configured count (#825 warns about that config)
[InlineData(10_000, 16_384, 16_384)]
public void The_count_ceiling_never_exceeds_the_agents_buffer(int configured, long bufferSize, int expected) =>
MTConnectDriver.SampleCountCeiling(configured, bufferSize).ShouldBe(expected);
[Fact]
public async Task A_behind_stream_stops_raising_at_the_agents_buffer_size()
{
var (driver, client, _) = await SubscribedAsync(commsLossConfirmMs: 0, sampleCount: 2, bufferSize: 5);
try
{
var next = Primed;
async Task Behind(int count)
{
for (var i = 0; i < 3; i++)
{
await client.PumpAsync(Chunk(next, count, last: 900)).WaitAsync(Watchdog, Ct);
next += count;
}
}
await Behind(2);
await client.WaitForSampleCallsAsync(2).WaitAsync(Watchdog, Ct);
await Behind(4);
await client.WaitForSampleCallsAsync(3).WaitAsync(Watchdog, Ct);
client.SampleCounts.ToArray().ShouldBe(new[] { 2, 4, 5 }, "the raise is clamped to bufferSize 5, never 8");
await Behind(5); // at the ceiling: still behind, but not reopened
await Task.Delay(200, Ct);
client.SampleCallCount.ShouldBe(3);
driver.GetDiagnostics()["sampleCount"].ShouldBe("5");
}
finally
{
await driver.ShutdownAsync(CancellationToken.None);
}
}
}