Joseph Doherty
19ffb3c902
- A /sample or re-baseline failure whose chain holds SocketError.ConnectionRefused, unbroken for CommsLossConfirmMs (new, default 2000; 0 = first refusal; negative = off), publishes every subscribed value BadNoCommunication once per outage; the first delivery afterwards republishes all of them from the index (unchanged values included). Timeouts/resets/DNS/pause keep the #527 watchdog contract. - SampleCountCeiling clamps the adaptive raise to the /probe bufferSize (never below SampleCount). - Docs: MTConnect.md comms-loss section + config key; drivers.md rule. Refs #846
230 lines
10 KiB
C#
230 lines
10 KiB
C#
using System.Collections.Concurrent;
|
||
using System.Net.Http;
|
||
using System.Net.Sockets;
|
||
using Shouldly;
|
||
using Xunit;
|
||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||
|
||
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
|
||
|
||
/// <summary>
|
||
/// lmxopcua#846 — a REFUSED Agent connection is a definite loss: after <c>CommsLossConfirmMs</c> of nothing but
|
||
/// refusals every subscribed value goes <c>BadNoCommunication</c> (instead of Good-but-stale until the ~90 s
|
||
/// watchdog), and the first delivery afterwards republishes every value Good — including unchanged ones. Ambiguous
|
||
/// failures (timeouts, a closed stream) keep the watchdog contract. Plus the raised-count ceiling is clamped to the
|
||
/// Agent's <c>bufferSize</c>.
|
||
/// </summary>
|
||
public sealed class MTConnectCommsLossTests
|
||
{
|
||
private const long Instance = 1655000000L;
|
||
private const long Primed = 108L;
|
||
private const uint BadNoCommunication = 0x80310000u;
|
||
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);
|
||
|
||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||
|
||
private static MTConnectStreamsResult Chunk(long first, int observations, long? last = null)
|
||
{
|
||
var obs = Enumerable.Range(0, observations)
|
||
.Select(i => new MTConnectObservation(
|
||
"dev1_pos", (first + i).ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||
new DateTime(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc).AddSeconds(first + i)))
|
||
.ToList();
|
||
return new MTConnectStreamsResult(Instance, first + observations, first, obs, last);
|
||
}
|
||
|
||
private static Exception Refused() =>
|
||
new HttpRequestException("Connection refused (fixture-agent:5000)", new SocketException((int)SocketError.ConnectionRefused));
|
||
|
||
private static async Task<(MTConnectDriver Driver, CannedAgentClient Client, ConcurrentQueue<DataChangeEventArgs> Seen)>
|
||
SubscribedAsync(int commsLossConfirmMs, int sampleCount = 1000, long? bufferSize = null)
|
||
{
|
||
var client = CannedAgentClient.FromFixtures();
|
||
if (bufferSize is { } size) client.Probe = client.Probe with { BufferSize = size };
|
||
var driver = new MTConnectDriver(
|
||
new MTConnectDriverOptions
|
||
{
|
||
AgentUri = "http://fixture-agent:5000",
|
||
RequestTimeoutMs = 5000,
|
||
SampleCount = sampleCount,
|
||
CommsLossConfirmMs = commsLossConfirmMs,
|
||
Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 0, MaxBackoffMs = 200 },
|
||
Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)],
|
||
},
|
||
"mt-846",
|
||
_ => client);
|
||
var seen = new ConcurrentQueue<DataChangeEventArgs>();
|
||
driver.OnDataChange += (_, e) => seen.Enqueue(e);
|
||
await driver.InitializeAsync("{}", Ct);
|
||
await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct);
|
||
await client.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
|
||
return (driver, client, seen);
|
||
}
|
||
|
||
private static async Task WaitUntil(Func<bool> condition, int timeoutMs = 10_000)
|
||
{
|
||
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
||
while (!condition())
|
||
{
|
||
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
|
||
await Task.Delay(20, Ct);
|
||
}
|
||
}
|
||
|
||
[Fact]
|
||
public void Only_a_refused_connection_is_a_definite_loss()
|
||
{
|
||
MTConnectDriver.IsDefiniteConnectionLoss(Refused()).ShouldBeTrue();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new SocketException((int)SocketError.ConnectionRefused)).ShouldBeTrue();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new AggregateException(Refused())).ShouldBeTrue();
|
||
|
||
MTConnectDriver.IsDefiniteConnectionLoss(null).ShouldBeFalse();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new TimeoutException("no answer")).ShouldBeFalse();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("reset", new SocketException((int)SocketError.ConnectionReset))).ShouldBeFalse();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("dns", new SocketException((int)SocketError.HostNotFound))).ShouldBeFalse();
|
||
MTConnectDriver.IsDefiniteConnectionLoss(new MTConnectStreamEndedException(MTConnectStreamEndReason.ConnectionClosed, "x", 3)).ShouldBeFalse();
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_refused_agent_marks_values_bad_at_once_and_the_first_delivery_republishes_them_good()
|
||
{
|
||
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0);
|
||
try
|
||
{
|
||
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
|
||
seen.Last().Snapshot.StatusCode.ShouldBe(0u);
|
||
|
||
client.SampleFailure = Refused();
|
||
client.EndStream(); // the connection drops; every reconnect is refused
|
||
await WaitUntil(() => seen.Last().Snapshot.StatusCode == BadNoCommunication);
|
||
driver.CommsLossMarked.ShouldBeTrue();
|
||
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||
var badCount = seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication);
|
||
await Task.Delay(400, Ct); // more refused attempts: marked ONCE per outage
|
||
seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(badCount);
|
||
|
||
// The Agent is back. An observation-free heartbeat is enough: the UNCHANGED value comes back Good.
|
||
client.SampleFailure = null;
|
||
var calls = client.SampleCallCount;
|
||
await client.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
|
||
await client.PumpAsync(Chunk(Primed + 1, 0)).WaitAsync(Watchdog, Ct);
|
||
await WaitUntil(() => seen.Last().Snapshot.StatusCode == 0u);
|
||
var recovered = seen.Last().Snapshot;
|
||
recovered.Value.ShouldBe(108.0);
|
||
driver.CommsLossMarked.ShouldBeFalse();
|
||
driver.GetHealth().State.ShouldBe(DriverState.Healthy);
|
||
}
|
||
finally
|
||
{
|
||
await driver.ShutdownAsync(CancellationToken.None);
|
||
}
|
||
}
|
||
|
||
[Fact]
|
||
public async Task An_ambiguous_failure_keeps_the_watchdog_contract()
|
||
{
|
||
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0);
|
||
try
|
||
{
|
||
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
|
||
client.SampleFailure = new TimeoutException("the Agent did not answer /sample in time");
|
||
var calls = client.SampleCallCount;
|
||
client.EndStream();
|
||
await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
|
||
seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication);
|
||
driver.CommsLossMarked.ShouldBeFalse();
|
||
driver.GetHealth().State.ShouldBe(DriverState.Degraded); // the watchdog still has its evidence
|
||
}
|
||
finally
|
||
{
|
||
await driver.ShutdownAsync(CancellationToken.None);
|
||
}
|
||
}
|
||
|
||
[Theory]
|
||
[InlineData(60_000)] // a confirmation window not yet elapsed
|
||
[InlineData(-1)] // the fast path disabled
|
||
public async Task Refusals_inside_the_confirmation_window_or_with_the_fast_path_off_mark_nothing(int confirmMs)
|
||
{
|
||
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: confirmMs);
|
||
try
|
||
{
|
||
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
|
||
client.SampleFailure = Refused();
|
||
var calls = client.SampleCallCount;
|
||
client.EndStream();
|
||
await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
|
||
seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication);
|
||
driver.CommsLossMarked.ShouldBeFalse();
|
||
}
|
||
finally
|
||
{
|
||
await driver.ShutdownAsync(CancellationToken.None);
|
||
}
|
||
}
|
||
|
||
[Fact]
|
||
public async Task A_short_confirmation_window_marks_once_the_refusals_have_lasted_it()
|
||
{
|
||
var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 300);
|
||
try
|
||
{
|
||
await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct);
|
||
client.SampleFailure = Refused();
|
||
var started = DateTime.UtcNow;
|
||
client.EndStream();
|
||
await WaitUntil(() => driver.CommsLossMarked, timeoutMs: 5000);
|
||
(DateTime.UtcNow - started).ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(280));
|
||
seen.Last().Snapshot.StatusCode.ShouldBe(BadNoCommunication);
|
||
}
|
||
finally
|
||
{
|
||
await driver.ShutdownAsync(CancellationToken.None);
|
||
}
|
||
}
|
||
|
||
// ---- the raised-count ceiling vs the Agent's bufferSize ----
|
||
|
||
[Theory]
|
||
[InlineData(1000, 0, 16_000)] // unknown buffer: the 16× rule
|
||
[InlineData(1000, 4096, 4096)] // a known buffer caps it
|
||
[InlineData(1000, 131_072, 16_000)] // a big buffer changes nothing
|
||
[InlineData(1000, 500, 1000)] // never below the configured count (#825 warns about that config)
|
||
[InlineData(10_000, 16_384, 16_384)]
|
||
public void The_count_ceiling_never_exceeds_the_agents_buffer(int configured, long bufferSize, int expected) =>
|
||
MTConnectDriver.SampleCountCeiling(configured, bufferSize).ShouldBe(expected);
|
||
|
||
[Fact]
|
||
public async Task A_behind_stream_stops_raising_at_the_agents_buffer_size()
|
||
{
|
||
var (driver, client, _) = await SubscribedAsync(commsLossConfirmMs: 0, sampleCount: 2, bufferSize: 5);
|
||
try
|
||
{
|
||
var next = Primed;
|
||
async Task Behind(int count)
|
||
{
|
||
for (var i = 0; i < 3; i++)
|
||
{
|
||
await client.PumpAsync(Chunk(next, count, last: 900)).WaitAsync(Watchdog, Ct);
|
||
next += count;
|
||
}
|
||
}
|
||
|
||
await Behind(2);
|
||
await client.WaitForSampleCallsAsync(2).WaitAsync(Watchdog, Ct);
|
||
await Behind(4);
|
||
await client.WaitForSampleCallsAsync(3).WaitAsync(Watchdog, Ct);
|
||
client.SampleCounts.ToArray().ShouldBe(new[] { 2, 4, 5 }, "the raise is clamped to bufferSize 5, never 8");
|
||
|
||
await Behind(5); // at the ceiling: still behind, but not reopened
|
||
await Task.Delay(200, Ct);
|
||
client.SampleCallCount.ShouldBe(3);
|
||
driver.GetDiagnostics()["sampleCount"].ShouldBe("5");
|
||
}
|
||
finally
|
||
{
|
||
await driver.ShutdownAsync(CancellationToken.None);
|
||
}
|
||
}
|
||
}
|