feat(mtconnect): CommsLossConfirmMs AdminUI field, refused-connection fast path reaches native conditions, dedicated live-gate Agent (#852)
- AdminUI MTConnect driver form: "Comms-loss confirm (ms)" in the Reconnect section, always written (0 = first refusal, negative = off is never dropped), case-insensitive load, unknown keys preserved, advisory at >= 90000 ms (the watchdog fires first). Form output byte-pinned and parsed through MTConnectDriver.ParseOptions on the driver side. - Driver: a confirmed refused-connection outage now annotates every told CONDITION quality-only BadNoCommunication (state/ack kept, once per outage, UNAVAILABLE-Bad skipped); the first delivery afterwards re-diffs all subscribed conditions so unchanged ones get a quality-only Good. - docker-dev/docker-compose.mtconnect-gate.yml: throwaway Agent in its adapter's network namespace so a stop is a genuine ConnectionRefused (a plain stopped container is a DNS failure = ambiguous). - Docs: MTConnect.md comms-loss section + live-gate recipe, docker-dev README, drivers.md rule. Refs #852
This commit is contained in:
+233
@@ -0,0 +1,233 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.Http;
|
||||
using System.Net.Sockets;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.MTConnect.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#852 — the #846 refused-connection fast path reaches native CONDITIONS too: once the refusals are
|
||||
/// confirmed every told condition gets a quality-only <c>BadNoCommunication</c> (state and ack untouched, no
|
||||
/// Raise/Clear), once per outage; the first delivery afterwards lifts it with a quality-only Good — even when that
|
||||
/// delivery does not mention the condition. Ambiguous failures stay with the watchdog.
|
||||
/// </summary>
|
||||
/// <remarks>Every absence is followed by an event that MUST arrive, asserted to be the next one.</remarks>
|
||||
public sealed class MTConnectConditionCommsLossTests
|
||||
{
|
||||
private const string ConditionRawPath = "Plant/MT/dev1/SystemCond";
|
||||
private const string ConditionId = "dev1_system_cond";
|
||||
private const long Instance = 1655000000L;
|
||||
private const uint BadNoCommunication = 0x80310000u;
|
||||
|
||||
private static readonly DateTime ObservedAt = new(2026, 11, 11, 12, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly TimeSpan Watchdog = TimeSpan.FromSeconds(30);
|
||||
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static Exception Refused() =>
|
||||
new HttpRequestException("Connection refused (fixture-agent:5000)", new SocketException((int)SocketError.ConnectionRefused));
|
||||
|
||||
private static MTConnectStreamsResult Chunk(long first, params MTConnectObservation[] observations) =>
|
||||
new(Instance, first + 1, first, observations);
|
||||
|
||||
private static MTConnectObservation Cond(string state, string? nativeCode = null, string? text = null) =>
|
||||
new(ConditionId, state, ObservedAt, Condition: new MTConnectConditionDetail(text, nativeCode, null, null, "SYSTEM"));
|
||||
|
||||
private static async Task<(MTConnectDriver Driver, CannedAgentClient Agent, ConcurrentQueue<AlarmEventArgs> Events)>
|
||||
SubscribedAsync()
|
||||
{
|
||||
var agent = CannedAgentClient.FromFixtures();
|
||||
var driver = new MTConnectDriver(
|
||||
new MTConnectDriverOptions
|
||||
{
|
||||
AgentUri = "http://fixture-agent:5000",
|
||||
RequestTimeoutMs = 5000,
|
||||
CommsLossConfirmMs = 0,
|
||||
Reconnect = new MTConnectReconnectOptions { MinBackoffMs = 0, MaxBackoffMs = 200 },
|
||||
RawTags = [new RawTagEntry(ConditionRawPath, $$"""{"address":"{{ConditionId}}"}""", false, "dev1")],
|
||||
},
|
||||
"mt-852",
|
||||
_ => agent);
|
||||
await driver.InitializeAsync("{}", Ct);
|
||||
|
||||
var events = new ConcurrentQueue<AlarmEventArgs>();
|
||||
driver.OnAlarmEvent += (_, e) => events.Enqueue(e);
|
||||
await driver.SubscribeAlarmsAsync([ConditionRawPath], Ct);
|
||||
await agent.WaitForSampleCallsAsync(1).WaitAsync(Watchdog, Ct);
|
||||
return (driver, agent, events);
|
||||
}
|
||||
|
||||
private static async Task WaitUntil(Func<bool> condition, int timeoutMs = 10_000)
|
||||
{
|
||||
var deadline = DateTime.UtcNow.AddMilliseconds(timeoutMs);
|
||||
while (!condition())
|
||||
{
|
||||
if (DateTime.UtcNow > deadline) throw new TimeoutException("condition not reached");
|
||||
await Task.Delay(20, Ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task RecoverAsync(CannedAgentClient agent)
|
||||
{
|
||||
agent.SampleFailure = null;
|
||||
var calls = agent.SampleCallCount;
|
||||
await agent.WaitForSampleCallsAsync(calls + 1).WaitAsync(Watchdog, Ct);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_refused_agent_annotates_an_active_condition_bad_once_and_the_first_delivery_lifts_it()
|
||||
{
|
||||
var (driver, agent, events) = await SubscribedAsync();
|
||||
try
|
||||
{
|
||||
await agent.PumpAsync(Chunk(108, Cond("Fault", "E17", "overtemp"))).WaitAsync(Watchdog, Ct);
|
||||
events.ShouldHaveSingleItem().Kind.ShouldBe(AlarmTransitionKind.Raise);
|
||||
|
||||
agent.SampleFailure = Refused();
|
||||
agent.EndStream();
|
||||
await WaitUntil(() => events.Count == 2);
|
||||
var bad = events.Last();
|
||||
bad.IsQualityOnly.ShouldBeTrue("the Active state and the ack must not move — quality only");
|
||||
bad.QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
bad.ConditionId.ShouldBe(ConditionRawPath);
|
||||
bad.OpcUaSeverity.ShouldBe(800);
|
||||
driver.CommsLossMarked.ShouldBeTrue();
|
||||
|
||||
await Task.Delay(400, Ct); // more refusals: annotated ONCE per outage
|
||||
events.Count.ShouldBe(2);
|
||||
|
||||
// The Agent is back; an observation-free heartbeat does not mention the condition, and must still lift it.
|
||||
await RecoverAsync(agent);
|
||||
await agent.PumpAsync(Chunk(109)).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => events.Count == 3);
|
||||
var good = events.Last();
|
||||
good.IsQualityOnly.ShouldBeTrue("an unchanged Fault is not re-raised (that would un-acknowledge it)");
|
||||
good.QualityStatusCode.ShouldBe(0u);
|
||||
|
||||
// Positive control for "nothing else was emitted": the next real transition is the next event.
|
||||
await agent.PumpAsync(Chunk(110, Cond("Normal"))).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => events.Count == 4);
|
||||
events.Last().Kind.ShouldBe(AlarmTransitionKind.Clear);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_inactive_told_condition_is_annotated_too()
|
||||
{
|
||||
var (driver, agent, events) = await SubscribedAsync();
|
||||
try
|
||||
{
|
||||
// Raise + Clear: the server holds a told Normal statement for the condition.
|
||||
await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
|
||||
await agent.PumpAsync(Chunk(109, Cond("Normal"))).WaitAsync(Watchdog, Ct);
|
||||
events.Count.ShouldBe(2);
|
||||
|
||||
agent.SampleFailure = Refused();
|
||||
agent.EndStream();
|
||||
await WaitUntil(() => events.Count == 3);
|
||||
events.Last().IsQualityOnly.ShouldBeTrue();
|
||||
events.Last().QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
|
||||
await RecoverAsync(agent);
|
||||
await agent.PumpAsync(Chunk(110)).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => events.Count == 4);
|
||||
events.Last().IsQualityOnly.ShouldBeTrue();
|
||||
events.Last().QualityStatusCode.ShouldBe(0u);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_condition_already_unavailable_is_not_re_annotated_and_stays_bad_on_recovery()
|
||||
{
|
||||
var (driver, agent, events) = await SubscribedAsync();
|
||||
try
|
||||
{
|
||||
await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
|
||||
await agent.PumpAsync(Chunk(109, Cond("UNAVAILABLE"))).WaitAsync(Watchdog, Ct);
|
||||
events.Count.ShouldBe(2);
|
||||
events.Last().QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
|
||||
agent.SampleFailure = Refused();
|
||||
agent.EndStream();
|
||||
await WaitUntil(() => driver.CommsLossMarked);
|
||||
|
||||
// The Agent answers, but the adapter is still UNAVAILABLE: no Good may be told.
|
||||
await RecoverAsync(agent);
|
||||
await agent.PumpAsync(Chunk(110)).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => !driver.CommsLossMarked);
|
||||
|
||||
// Positive control: the adapter returns with the same Fault — quality lifted, NOT re-raised — and that
|
||||
// is the very next event after the UNAVAILABLE annotation.
|
||||
await agent.PumpAsync(Chunk(111, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => events.Count >= 3);
|
||||
events.Count.ShouldBe(3);
|
||||
events.Last().IsQualityOnly.ShouldBeTrue();
|
||||
events.Last().QualityStatusCode.ShouldBe(0u);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_ambiguous_failure_annotates_no_condition()
|
||||
{
|
||||
var (driver, agent, events) = await SubscribedAsync();
|
||||
try
|
||||
{
|
||||
await agent.PumpAsync(Chunk(108, Cond("Fault", "E17"))).WaitAsync(Watchdog, Ct);
|
||||
events.Count.ShouldBe(1);
|
||||
|
||||
agent.SampleFailure = new TimeoutException("the Agent did not answer /sample in time");
|
||||
var calls = agent.SampleCallCount;
|
||||
agent.EndStream();
|
||||
await agent.WaitForSampleCallsAsync(calls + 3).WaitAsync(Watchdog, Ct);
|
||||
driver.CommsLossMarked.ShouldBeFalse();
|
||||
|
||||
// Positive control: the next event is the real Clear, not a quality annotation.
|
||||
await RecoverAsync(agent);
|
||||
await agent.PumpAsync(Chunk(109, Cond("Normal"))).WaitAsync(Watchdog, Ct);
|
||||
await WaitUntil(() => events.Count >= 2);
|
||||
events.Count.ShouldBe(2);
|
||||
events.Last().Kind.ShouldBe(AlarmTransitionKind.Clear);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await driver.ShutdownAsync(CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CommsLost_keeps_the_told_statement_and_says_nothing_twice()
|
||||
{
|
||||
var handle = new MTConnectAlarmSubscriptionHandle(1);
|
||||
var warning = new MTConnectConditionState(
|
||||
MTConnectConditionLevel.Warning, new MTConnectConditionDetail("coolant", "W1", null, null, "SYSTEM"), ObservedAt, 1);
|
||||
var told = new MTConnectToldCondition(warning, QualityBad: false);
|
||||
var at = ObservedAt.AddMinutes(5);
|
||||
|
||||
var (evt, after) = MTConnectConditionAlarmMapper.CommsLost(told, ConditionRawPath, ConditionId, handle, at);
|
||||
|
||||
evt.ShouldNotBeNull();
|
||||
evt!.IsQualityOnly.ShouldBeTrue();
|
||||
evt.QualityStatusCode.ShouldBe(BadNoCommunication);
|
||||
evt.SourceTimestampUtc.ShouldBe(at);
|
||||
evt.OpcUaSeverity.ShouldBe(500);
|
||||
after.ShouldBe(told with { QualityBad = true });
|
||||
|
||||
MTConnectConditionAlarmMapper.CommsLost(after, ConditionRawPath, ConditionId, handle, at).Event.ShouldBeNull();
|
||||
MTConnectConditionAlarmMapper.CommsLost(
|
||||
new MTConnectToldCondition(null, QualityBad: true), ConditionRawPath, ConditionId, handle, at).Event.ShouldBeNull();
|
||||
}
|
||||
}
|
||||
@@ -394,7 +394,7 @@ public sealed class MTConnectFactoryTests
|
||||
public void The_document_the_AdminUI_form_emits_arms_write_back()
|
||||
{
|
||||
const string formEmitted =
|
||||
"""{"agentUri":"http://agent:5000","requestTimeoutMs":5000,"sampleIntervalMs":1000,"sampleCount":1000,"heartbeatMs":10000,"probe":{"enabled":true,"intervalMs":5000,"timeoutMs":2000},"reconnect":{"minBackoffMs":0,"maxBackoffMs":30000,"backoffMultiplier":2},"write":{"enabled":true}}""";
|
||||
"""{"agentUri":"http://agent:5000","requestTimeoutMs":5000,"sampleIntervalMs":1000,"sampleCount":1000,"heartbeatMs":10000,"commsLossConfirmMs":2000,"probe":{"enabled":true,"intervalMs":5000,"timeoutMs":2000},"reconnect":{"minBackoffMs":0,"maxBackoffMs":30000,"backoffMultiplier":2},"write":{"enabled":true}}""";
|
||||
|
||||
var options = MTConnectDriver.ParseOptions("mt1", formEmitted);
|
||||
|
||||
@@ -407,8 +407,29 @@ public sealed class MTConnectFactoryTests
|
||||
"mt1",
|
||||
formEmitted.Replace("""enabled":true}""", """enabled":false}""", StringComparison.Ordinal))
|
||||
.Write.Enabled.ShouldBeFalse();
|
||||
options.CommsLossConfirmMs.ShouldBe(2000);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// lmxopcua#852: the AdminUI form's <c>commsLossConfirmMs</c> field binds through this parse as a plain
|
||||
/// integer in every meaningful state — default, 0 (first refusal) and negative (fast path off). The literals are
|
||||
/// the form's own output (byte-pinned on the form side by
|
||||
/// <c>MTConnectDriverFormModelTests.The_comms_loss_documents_are_the_literals_the_driver_test_parses</c>); an
|
||||
/// absent key keeps the driver default.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData(CommsLossOffDocument, -1)]
|
||||
[InlineData(CommsLossImmediateDocument, 0)]
|
||||
[InlineData("""{"agentUri":"http://agent:5000"}""", 2000)]
|
||||
public void The_comms_loss_window_the_AdminUI_form_emits_binds(string formEmitted, int expected) =>
|
||||
MTConnectDriver.ParseOptions("mt1", formEmitted).CommsLossConfirmMs.ShouldBe(expected);
|
||||
|
||||
private const string CommsLossOffDocument =
|
||||
"""{"agentUri":"http://agent:5000","requestTimeoutMs":5000,"sampleIntervalMs":1000,"sampleCount":1000,"heartbeatMs":10000,"commsLossConfirmMs":-1,"probe":{"enabled":true,"intervalMs":5000,"timeoutMs":2000},"reconnect":{"minBackoffMs":0,"maxBackoffMs":30000,"backoffMultiplier":2},"write":{"enabled":false}}""";
|
||||
|
||||
private const string CommsLossImmediateDocument =
|
||||
"""{"agentUri":"http://agent:5000","requestTimeoutMs":5000,"sampleIntervalMs":1000,"sampleCount":1000,"heartbeatMs":10000,"commsLossConfirmMs":0,"probe":{"enabled":true,"intervalMs":5000,"timeoutMs":2000},"reconnect":{"minBackoffMs":0,"maxBackoffMs":30000,"backoffMultiplier":2},"write":{"enabled":false}}""";
|
||||
|
||||
/// <summary>
|
||||
/// A malformed <c>write</c> block fails <b>exactly as its siblings do</b> — the whole
|
||||
/// document is one <c>JsonSerializer.Deserialize</c>, so a non-boolean <c>enabled</c> faults
|
||||
|
||||
Reference in New Issue
Block a user