using System.Collections.Concurrent; using System.Net.Http; using System.Net.Sockets; using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests; /// /// lmxopcua#846 — a REFUSED Agent connection is a definite loss: after CommsLossConfirmMs of nothing but /// refusals every subscribed value goes BadNoCommunication (instead of Good-but-stale until the ~90 s /// watchdog), and the first delivery afterwards republishes every value Good — including unchanged ones. Ambiguous /// failures (timeouts, a closed stream) keep the watchdog contract. Plus the raised-count ceiling is clamped to the /// Agent's bufferSize. /// public sealed class MTConnectCommsLossTests { private const long Instance = 1655000000L; private const long Primed = 108L; private const uint BadNoCommunication = 0x80310000u; private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30); private static CancellationToken Ct => TestContext.Current.CancellationToken; private static MTConnectStreamsResult Chunk(long first, int observations, long? last = null) { var obs = Enumerable.Range(0, observations) .Select(i => new MTConnectObservation( "dev1_pos", (first + i).ToString(System.Globalization.CultureInfo.InvariantCulture), new DateTime(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc).AddSeconds(first + i))) .ToList(); return new MTConnectStreamsResult(Instance, first + observations, first, obs, last); } private static Exception Refused() => new HttpRequestException("Connection refused (fixture-agent:5000)", new SocketException((int)SocketError.ConnectionRefused)); private static async Task<(MTConnectDriver Driver, CannedAgentClient Client, ConcurrentQueue Seen)> SubscribedAsync(int commsLossConfirmMs, int sampleCount = 1000, long? bufferSize = null) { var client = CannedAgentClient.FromFixtures(); if (bufferSize is { } size) client.Probe = client.Probe with { BufferSize = size }; var driver = new MTConnectDriver( new MTConnectDriverOptions { AgentUri = "http://fixture-agent:5000", RequestTimeoutMs = 5000, SampleCount = sampleCount, CommsLossConfirmMs = commsLossConfirmMs, Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 0, MaxBackoffMs = 200 }, Tags = [new MTConnectTagDefinition("dev1_pos", DriverDataType.Float64)], }, "mt-846", _ => client); var seen = new ConcurrentQueue(); driver.OnDataChange += (_, e) => seen.Enqueue(e); await driver.InitializeAsync("{}", Ct); await driver.SubscribeAsync(["dev1_pos"], TimeSpan.FromMilliseconds(50), Ct); await client.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct); return (driver, client, seen); } private static async Task WaitUntil(Func condition, int timeoutMs = 10_000) { var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs); while (!condition()) { if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached"); await Task.Delay(20, Ct); } } [Fact] public void Only_a_refused_connection_is_a_definite_loss() { MTConnectDriver.IsDefiniteConnectionLoss(Refused()).ShouldBeTrue(); MTConnectDriver.IsDefiniteConnectionLoss(new SocketException((int)SocketError.ConnectionRefused)).ShouldBeTrue(); MTConnectDriver.IsDefiniteConnectionLoss(new AggregateException(Refused())).ShouldBeTrue(); MTConnectDriver.IsDefiniteConnectionLoss(null).ShouldBeFalse(); MTConnectDriver.IsDefiniteConnectionLoss(new TimeoutException("no answer")).ShouldBeFalse(); MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("reset", new SocketException((int)SocketError.ConnectionReset))).ShouldBeFalse(); MTConnectDriver.IsDefiniteConnectionLoss(new HttpRequestException("dns", new SocketException((int)SocketError.HostNotFound))).ShouldBeFalse(); MTConnectDriver.IsDefiniteConnectionLoss(new MTConnectStreamEndedException(MTConnectStreamEndReason.ConnectionClosed, "x", 3)).ShouldBeFalse(); } [Fact] public async Task A_refused_agent_marks_values_bad_at_once_and_the_first_delivery_republishes_them_good() { var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0); try { await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct); seen.Last().Snapshot.StatusCode.ShouldBe(0u); client.SampleFailure = Refused(); client.EndStream(); // the connection drops; every reconnect is refused await WaitUntil(() => seen.Last().Snapshot.StatusCode == BadNoCommunication); driver.CommsLossMarked.ShouldBeTrue(); driver.GetHealth().State.ShouldBe(DriverState.Degraded); var badCount = seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication); await Task.Delay(400, Ct); // more refused attempts: marked ONCE per outage seen.Count(e => e.Snapshot.StatusCode == BadNoCommunication).ShouldBe(badCount); // The Agent is back. An observation-free heartbeat is enough: the UNCHANGED value comes back Good. client.SampleFailure = null; var calls = client.SampleCallCount; await client.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct); await client.PumpAsync(Chunk(Primed + 1, 0)).WaitAsync(Watchdog, Ct); await WaitUntil(() => seen.Last().Snapshot.StatusCode == 0u); var recovered = seen.Last().Snapshot; recovered.Value.ShouldBe(108.0); driver.CommsLossMarked.ShouldBeFalse(); driver.GetHealth().State.ShouldBe(DriverState.Healthy); } finally { await driver.ShutdownAsync(CancellationToken.None); } } [Fact] public async Task An_ambiguous_failure_keeps_the_watchdog_contract() { var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 0); try { await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct); client.SampleFailure = new TimeoutException("the Agent did not answer /sample in time"); var calls = client.SampleCallCount; client.EndStream(); await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct); seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication); driver.CommsLossMarked.ShouldBeFalse(); driver.GetHealth().State.ShouldBe(DriverState.Degraded); // the watchdog still has its evidence } finally { await driver.ShutdownAsync(CancellationToken.None); } } [Theory] [InlineData(60_000)] // a confirmation window not yet elapsed [InlineData(-1)] // the fast path disabled public async Task Refusals_inside_the_confirmation_window_or_with_the_fast_path_off_mark_nothing(int confirmMs) { var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: confirmMs); try { await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct); client.SampleFailure = Refused(); var calls = client.SampleCallCount; client.EndStream(); await client.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct); seen.ShouldNotContain(e => e.Snapshot.StatusCode == BadNoCommunication); driver.CommsLossMarked.ShouldBeFalse(); } finally { await driver.ShutdownAsync(CancellationToken.None); } } [Fact] public async Task A_short_confirmation_window_marks_once_the_refusals_have_lasted_it() { var (driver, client, seen) = await SubscribedAsync(commsLossConfirmMs: 300); try { await client.PumpAsync(Chunk(Primed, 1)).WaitAsync(Watchdog, Ct); client.SampleFailure = Refused(); var started = DateTime.UtcNow; client.EndStream(); await WaitUntil(() => driver.CommsLossMarked, timeoutMs: 5000); (DateTime.UtcNow - started).ShouldBeGreaterThanOrEqualTo(TimeSpan.FromMilliseconds(280)); seen.Last().Snapshot.StatusCode.ShouldBe(BadNoCommunication); } finally { await driver.ShutdownAsync(CancellationToken.None); } } // ---- the raised-count ceiling vs the Agent's bufferSize ---- [Theory] [InlineData(1000, 0, 16_000)] // unknown buffer: the 16× rule [InlineData(1000, 4096, 4096)] // a known buffer caps it [InlineData(1000, 131_072, 16_000)] // a big buffer changes nothing [InlineData(1000, 500, 1000)] // never below the configured count (#825 warns about that config) [InlineData(10_000, 16_384, 16_384)] public void The_count_ceiling_never_exceeds_the_agents_buffer(int configured, long bufferSize, int expected) => MTConnectDriver.SampleCountCeiling(configured, bufferSize).ShouldBe(expected); [Fact] public async Task A_behind_stream_stops_raising_at_the_agents_buffer_size() { var (driver, client, _) = await SubscribedAsync(commsLossConfirmMs: 0, sampleCount: 2, bufferSize: 5); try { var next = Primed; async Task Behind(int count) { for (var i = 0; i < 3; i++) { await client.PumpAsync(Chunk(next, count, last: 900)).WaitAsync(Watchdog, Ct); next += count; } } await Behind(2); await client.WaitForSampleCallsAsync(2).WaitAsync(Watchdog, Ct); await Behind(4); await client.WaitForSampleCallsAsync(3).WaitAsync(Watchdog, Ct); client.SampleCounts.ToArray().ShouldBe(new[] { 2, 4, 5 }, "the raise is clamped to bufferSize 5, never 8"); await Behind(5); // at the ceiling: still behind, but not reopened await Task.Delay(200, Ct); client.SampleCallCount.ShouldBe(3); driver.GetDiagnostics()["sampleCount"].ShouldBe("5"); } finally { await driver.ShutdownAsync(CancellationToken.None); } } }