From 530f1b2f0b3320ea2b66d4de1f82dfd084f0d8a4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:02:11 -0400 Subject: [PATCH 01/34] fix(modbus): transport self-heals with one throttled in-gate connect; PDU length check P3: a torn-down socket is re-dialled inside the gate (one attempt per backoff window) instead of failing fast with 'Transport not connected' until the 90 s comms-loss watchdog re-initialised the driver. L3: ConnectAsync disposes the previous/failed TcpClient, DNS runs under the per-op timeout, and the reconnect never sleeps inside the gate (the old 10-attempt backoff held it for ~81 s). H9: an MBAP response with no PDU, or an exception PDU with no code, is a TruncatedFrame desync rather than an IndexOutOfRangeException. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusRtuOverTcpTransport.cs | 27 +-- .../ModbusSocketLifecycle.cs | 145 ++++++++++------ .../ModbusTcpTransport.cs | 42 +++-- .../ModbusTransportUnavailableException.cs | 23 +++ .../CommsRecoveryTests.cs | 88 ++++++++++ .../TcpRelay.cs | 101 +++++++++++ .../ModbusTransportSelfHealTests.cs | 139 +++++++++++++++ .../ScriptedModbusTcpServer.cs | 160 ++++++++++++++++++ 8 files changed, 655 insertions(+), 70 deletions(-) create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTransportUnavailableException.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/TcpRelay.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTransportSelfHealTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedModbusTcpServer.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs index d159e24a..69ccf915 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs @@ -91,7 +91,7 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport public async Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) { if (_disposed) throw new ObjectDisposedException(nameof(ModbusRtuOverTcpTransport)); - if (CurrentStream is null) throw new InvalidOperationException("Transport not connected"); + if (_life is null && _testStream is null) throw new InvalidOperationException("Transport not connected"); // Single-flight: RTU carries no transaction id, so at most one transaction may be on the bus // at a time — serialise every send/receive under the gate. @@ -101,11 +101,14 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport // Proactive idle-disconnect (real socket only): if the socket has been quiet longer than // the configured threshold, tear it down + reconnect before this PDU lands. Defends // against silent NAT / firewall reaping. - if (_life is not null && _life.ShouldReconnectForIdle()) - { + if (_life is not null && _life.Stream is not null && _life.ShouldReconnectForIdle()) await _life.TearDownAsync().ConfigureAwait(false); - await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false); - } + + // Self-heal (real socket only): a socket torn down by an earlier failure is re-dialled here, + // once, throttled by the backoff window — same contract as ModbusTcpTransport. + var freshlyConnected = _life is not null && _life.Stream is null; + if (freshlyConnected) + await _life!.EnsureConnectedAsync(ct).ConfigureAwait(false); try { @@ -113,14 +116,16 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport _life?.MarkSuccess(); return result; } - catch (Exception ex) when (_life is not null && _life.AutoReconnect && ModbusSocketLifecycle.IsSocketLevelFailure(ex)) + catch (Exception ex) when (!freshlyConnected && _life is not null && _life.AutoReconnect + && !ct.IsCancellationRequested + && ModbusSocketLifecycle.IsSocketLevelFailure(ex)) { - // Mid-transaction drop: tear down the dead socket, reconnect (with backoff if - // configured), resend. Single retry — a second failure propagates so health/status - // reflect reality. Never reached in the ForTest seam (a raw injected stream has no - // lifecycle to reconnect). + // Mid-transaction drop: tear down the dead socket, make ONE throttled reconnect attempt + // (never a sleep inside the gate), resend. Single retry — a second failure propagates so + // health/status reflect reality. Never reached in the ForTest seam (a raw injected stream + // has no lifecycle to reconnect). await _life.TearDownAsync().ConfigureAwait(false); - await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false); + await _life.EnsureConnectedAsync(ct).ConfigureAwait(false); var result = await SendOnceAsync(unitId, pdu, ct).ConfigureAwait(false); _life.MarkSuccess(); return result; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs index 024d28f9..7a032cdd 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs @@ -4,8 +4,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus; /// /// Owns the raw TCP socket lifecycle shared by every Modbus-over-TCP transport (MBAP and -/// RTU-over-TCP alike): the IPv4-preference connect, OS-level keep-alive, geometric-backoff -/// reconnect, idle-disconnect tracking, and teardown. It exposes the current +/// RTU-over-TCP alike): the IPv4-preference connect, OS-level keep-alive, throttled +/// reconnect (one attempt per geometric-backoff window), idle-disconnect tracking, and teardown. It exposes the current /// so the composing transport can drive its own framing on top — /// the lifecycle knows nothing about MBAP / RTU wire layout. /// @@ -14,7 +14,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus; /// Extracted verbatim from so the connect / reconnect / /// keep-alive / idle behaviour is written once and composed by both transports. The /// constructor is connection-free — all socket I/O happens in / -/// . +/// (one throttled attempt per call — never a sleep loop). /// /// /// Why keep-alive matters for DL205/DL260: the AutomationDirect H2-ECOM100 does NOT send @@ -37,6 +37,9 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable private NetworkStream? _stream; private DateTime _lastSuccessUtc = DateTime.UtcNow; + /// Monotonic millisecond clock for the reconnect throttle (test seam). + internal Func NowMs { get; set; } = () => Environment.TickCount64; + /// Initializes a new instance of the class. /// The host address or hostname of the Modbus server. /// The TCP port of the Modbus server. @@ -71,63 +74,107 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable /// A task representing the asynchronous connection operation. public async Task ConnectAsync(CancellationToken ct) { - // Resolve the host explicitly + prefer IPv4. .NET's TcpClient default-constructor is - // dual-stack (IPv6 first, fallback to IPv4) — but most Modbus TCP devices (PLCs and - // simulators like pymodbus) bind 0.0.0.0 only, so the IPv6 attempt times out and we - // burn the entire ConnectAsync budget before even trying IPv4. Resolving first + - // dialing the IPv4 address directly sidesteps that. - var addresses = await System.Net.Dns.GetHostAddressesAsync(_host, ct).ConfigureAwait(false); - var ipv4 = System.Linq.Enumerable.FirstOrDefault(addresses, - a => a.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork); - var target = ipv4 ?? (addresses.Length > 0 ? addresses[0] : System.Net.IPAddress.Loopback); - - _client = new TcpClient(target.AddressFamily); - EnableKeepAlive(_client, _keepAlive); + // A previous socket (failed, torn down or idle-reaped) is released FIRST: overwriting _client + // used to strand the old TcpClient for the finalizer — up to nine per backoff loop. + await TearDownAsync().ConfigureAwait(false); + // DNS resolution runs under the same per-op deadline as the TCP connect: an unresolvable or + // slow resolver must not hold the transport gate for the OS resolver's own (much longer) timeout. using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); cts.CancelAfter(_timeout); - await _client.ConnectAsync(target, _port, cts.Token).ConfigureAwait(false); - _stream = _client.GetStream(); - _lastSuccessUtc = DateTime.UtcNow; + TcpClient? client = null; + try + { + // Resolve the host explicitly + prefer IPv4. .NET's TcpClient default-constructor is + // dual-stack (IPv6 first, fallback to IPv4) — but most Modbus TCP devices (PLCs and + // simulators like pymodbus) bind 0.0.0.0 only, so the IPv6 attempt times out and we + // burn the entire ConnectAsync budget before even trying IPv4. Resolving first + + // dialing the IPv4 address directly sidesteps that. + var addresses = await System.Net.Dns.GetHostAddressesAsync(_host, cts.Token).ConfigureAwait(false); + var ipv4 = System.Linq.Enumerable.FirstOrDefault(addresses, + a => a.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork); + var target = ipv4 ?? (addresses.Length > 0 ? addresses[0] : System.Net.IPAddress.Loopback); + + client = new TcpClient(target.AddressFamily); + EnableKeepAlive(client, _keepAlive); + await client.ConnectAsync(target, _port, cts.Token).ConfigureAwait(false); + _stream = client.GetStream(); + _client = client; + client = null; // ownership transferred + _lastSuccessUtc = DateTime.UtcNow; + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // The per-op deadline, not the caller: normalise to a socket-level failure so every + // classifier treats a connect timeout as a comms fault. + throw new ModbusTransportUnavailableException( + $"Modbus connect to {_host}:{_port} timed out after {_timeout.TotalMilliseconds:0}ms"); + } + finally + { + // A failed attempt disposes its own client — never leave it for the finalizer. + client?.Dispose(); + } } + // Reconnect throttle state. Guarded by the composing transport's single-flight gate (every caller of + // EnsureConnectedAsync holds it), so no extra lock is needed. + private long _connectNotBeforeMs; + private TimeSpan _failureDelay = TimeSpan.Zero; + /// - /// Connect attempt with the configured geometric backoff. The first attempt fires after - /// (default zero — immediate); each - /// subsequent attempt sleeps for the previous delay times BackoffMultiplier, - /// capped at MaxDelay. Caller's cancellation token aborts the loop. + /// Ensures a live socket for the next transaction with AT MOST ONE connect attempt, throttled by the + /// configured geometric backoff: after a failed attempt, calls inside the backoff window fail fast + /// with and never dial. The window starts at + /// (floored at 1 s) and grows by + /// BackoffMultiplier per consecutive failure up to MaxDelay; a successful connect + /// resets it. Never sleeps — the composing transport calls it inside its gate, and a sleep there + /// used to queue every read, write and probe behind one dead socket for up to ~81 s. /// /// A cancellation token to observe for cancellation. - /// A task representing the asynchronous reconnect operation. - public async Task ConnectWithBackoffAsync(CancellationToken ct) + /// A task that completes when a socket is live. + /// Auto-reconnect is off, or the call is inside the backoff window. + public async Task EnsureConnectedAsync(CancellationToken ct) { - var delay = _reconnect.InitialDelay; - var attempt = 0; - while (true) + if (_stream is not null) return; + if (!_autoReconnect) + throw new ModbusTransportUnavailableException( + $"Modbus transport to {_host}:{_port} is not connected and auto-reconnect is disabled"); + + var now = NowMs(); + if (now < _connectNotBeforeMs) + throw new ModbusTransportUnavailableException( + $"Modbus reconnect to {_host}:{_port} throttled for another {_connectNotBeforeMs - now}ms after a failed connect"); + + try { - if (delay > TimeSpan.Zero) - await Task.Delay(delay, ct).ConfigureAwait(false); - try - { - await ConnectAsync(ct).ConfigureAwait(false); - return; - } - catch (Exception ex) when (IsSocketLevelFailure(ex) && _autoReconnect) - { - attempt++; - // Geometric growth, capped. Use Math.Min on ticks so we don't overflow with - // pathological multipliers / long deployments. - var nextTicks = (long)(Math.Max(delay.Ticks, TimeSpan.FromMilliseconds(100).Ticks) * _reconnect.BackoffMultiplier); - delay = TimeSpan.FromTicks(Math.Min(nextTicks, _reconnect.MaxDelay.Ticks)); - if (attempt >= 10) - { - // Bail after 10 attempts to surface persistent failure to the caller. With - // the default backoff (1s base, 2.0x mult, 30s cap) this is roughly 4 minutes - // of attempts; with InitialDelay=0 it's immediate up to the same cap. - throw; - } - } + await ConnectAsync(ct).ConfigureAwait(false); + _failureDelay = TimeSpan.Zero; + _connectNotBeforeMs = 0; } + catch (Exception) when (!ct.IsCancellationRequested) + { + _failureDelay = NextDelay(_failureDelay); + _connectNotBeforeMs = NowMs() + (long)_failureDelay.TotalMilliseconds; + throw; + } + } + + /// Minimum reconnect window after a failed connect — a dead device is dialled at most once per second. + private static readonly TimeSpan MinReconnectWindow = TimeSpan.FromSeconds(1); + + private TimeSpan NextDelay(TimeSpan previous) + { + if (previous <= TimeSpan.Zero) + { + var first = _reconnect.InitialDelay > MinReconnectWindow ? _reconnect.InitialDelay : MinReconnectWindow; + return first < _reconnect.MaxDelay ? first : _reconnect.MaxDelay > TimeSpan.Zero ? _reconnect.MaxDelay : first; + } + // Geometric growth, capped. Math.Min on ticks so a pathological multiplier cannot overflow. + var multiplier = _reconnect.BackoffMultiplier >= 1.0 ? _reconnect.BackoffMultiplier : 1.0; + var nextTicks = previous.Ticks * multiplier; + var cap = _reconnect.MaxDelay > TimeSpan.Zero ? _reconnect.MaxDelay.Ticks : long.MaxValue; + return TimeSpan.FromTicks(nextTicks >= cap ? cap : (long)nextTicks); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs index f6f94539..26534610 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs @@ -56,6 +56,9 @@ public sealed class ModbusTcpTransport : IModbusTransport _life = new ModbusSocketLifecycle(host, port, timeout, autoReconnect, keepAlive, idleDisconnect, reconnect); } + /// The socket lifecycle this transport composes (test seam for its clock). + internal ModbusSocketLifecycle Lifecycle => _life; + /// public Task ConnectAsync(CancellationToken ct) => _life.ConnectAsync(ct); @@ -63,7 +66,6 @@ public sealed class ModbusTcpTransport : IModbusTransport public async Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) { if (_disposed) throw new ObjectDisposedException(nameof(ModbusTcpTransport)); - if (_life.Stream is null) throw new InvalidOperationException("Transport not connected"); await _gate.WaitAsync(ct).ConfigureAwait(false); try @@ -72,11 +74,16 @@ public sealed class ModbusTcpTransport : IModbusTransport // threshold, tear it down + reconnect before this PDU lands. Defends against silent // NAT / firewall reaping where the socket looks alive locally but the upstream side // dropped it minutes ago. - if (_life.ShouldReconnectForIdle()) - { + if (_life.Stream is not null && _life.ShouldReconnectForIdle()) await _life.TearDownAsync().ConfigureAwait(false); - await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false); - } + + // Self-heal: a socket torn down by an earlier failure (two lost replies in a row, a refused + // reconnect) is re-dialled HERE, once, throttled by the backoff window. This used to throw + // "Transport not connected" before the gate, so the transport never recovered on its own and + // every call failed fast until the 90 s comms-loss watchdog re-initialised the whole driver. + var freshlyConnected = _life.Stream is null; + if (freshlyConnected) + await _life.EnsureConnectedAsync(ct).ConfigureAwait(false); try { @@ -84,13 +91,16 @@ public sealed class ModbusTcpTransport : IModbusTransport _life.MarkSuccess(); return result; } - catch (Exception ex) when (_life.AutoReconnect && ModbusSocketLifecycle.IsSocketLevelFailure(ex)) + catch (Exception ex) when (!freshlyConnected && _life.AutoReconnect + && !ct.IsCancellationRequested + && ModbusSocketLifecycle.IsSocketLevelFailure(ex)) { - // Mid-transaction drop: tear down the dead socket, reconnect (with backoff if - // configured), resend. Single retry — if it fails again, let it propagate so - // health/status reflect reality. + // Mid-transaction drop on a socket that had been live: tear down the dead socket, make ONE + // throttled reconnect attempt (never a sleep inside the gate), resend. Single retry — if it + // fails again, let it propagate so health/status reflect reality. A socket this call just + // dialled is not retried: it proved nothing about a stale connection. await _life.TearDownAsync().ConfigureAwait(false); - await _life.ConnectWithBackoffAsync(ct).ConfigureAwait(false); + await _life.EnsureConnectedAsync(ct).ConfigureAwait(false); var result = await SendOnceAsync(unitId, pdu, ct).ConfigureAwait(false); _life.MarkSuccess(); return result; @@ -162,10 +172,22 @@ public sealed class ModbusTcpTransport : IModbusTransport var respPdu = new byte[respLen - 1]; await ReadExactlyAsync(stream, respPdu, cts.Token).ConfigureAwait(false); + // Length check before indexing: a PDU must carry at least its function code, and an exception + // PDU its exception code too. Indexing past the end used to throw IndexOutOfRangeException — + // not an IOException, so no teardown/retry, and a malformed reply read as a driver bug. + if (respPdu.Length < 1) + throw new ModbusTransportDesyncException( + ModbusTransportDesyncException.DesyncReason.TruncatedFrame, + "Modbus response carried no PDU (MBAP length 1)"); + // Exception PDU: function code has high bit set. This is a well-formed protocol-level // error — the socket is still coherent, so it MUST propagate without teardown. if ((respPdu[0] & 0x80) != 0) { + if (respPdu.Length < 2) + throw new ModbusTransportDesyncException( + ModbusTransportDesyncException.DesyncReason.TruncatedFrame, + $"Modbus exception response fc=0x{respPdu[0]:X2} carried no exception code"); var fc = (byte)(respPdu[0] & 0x7F); var ex = respPdu[1]; throw new ModbusException(fc, ex, $"Modbus exception fc={fc} code={ex}"); diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTransportUnavailableException.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTransportUnavailableException.cs new file mode 100644 index 00000000..0df33d1b --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTransportUnavailableException.cs @@ -0,0 +1,23 @@ +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus; + +/// +/// Raised when a Modbus transport has no live socket and cannot get one for this call: the one +/// in-gate connect attempt failed, the reconnect is inside its backoff window, or auto-reconnect is +/// disabled. It says the ENDPOINT is unreachable, so the driver fails the rest of the batch fast +/// instead of paying the same failure once per tag. +/// +/// +/// Derives from so every existing socket-level classifier +/// (ModbusSocketLifecycle.IsSocketLevelFailure, the driver's comms-fault mapping) treats it as +/// a communication failure without extra plumbing. +/// +public sealed class ModbusTransportUnavailableException : IOException +{ + /// Initializes a new instance of the class. + /// A human-readable description. + /// The originating exception, if any. + public ModbusTransportUnavailableException(string message, Exception? inner = null) + : base(message, inner) + { + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs new file mode 100644 index 00000000..37192239 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs @@ -0,0 +1,88 @@ +using System.Diagnostics; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests; + +/// +/// Comms-loss → recovery against the real pymodbus standard fixture, with a loopback +/// in front of it so the device can disappear and come back without touching the +/// shared container. Covers the review 07 High findings P3 (transport self-heal) and H2 (deadband +/// recovery publish). +/// +[Collection(ModbusSimulatorCollection.Name)] +[Trait("Category", "Integration")] +[Trait("Device", "Standard")] +public sealed class CommsRecoveryTests(ModbusSimulatorFixture sim) +{ + private const uint StatusGood = 0u; + + private void SkipUnlessStandardLive() + { + if (sim.SkipReason is not null) Assert.Skip(sim.SkipReason); + var profile = Environment.GetEnvironmentVariable("MODBUS_SIM_PROFILE"); + if (profile is not null && !string.Equals(profile, "standard", StringComparison.OrdinalIgnoreCase)) + Assert.Skip($"MODBUS_SIM_PROFILE={profile} — these tests need the standard profile (HR[0..31] = address)."); + } + + private ModbusDriverOptions Options(TcpRelay relay, params ModbusTagDefinition[] tags) => new() + { + Host = "127.0.0.1", + Port = relay.Port, + UnitId = 1, + Timeout = TimeSpan.FromSeconds(2), + RawTags = ModbusRawTags.Entries(tags), + Probe = new ModbusProbeOptions { Enabled = false }, + }; + + private static async Task WaitFor(Func> probe, Func done, TimeSpan budget) + { + var sw = Stopwatch.StartNew(); + T last; + do + { + last = await probe(); + if (done(last)) return last; + await Task.Delay(200, TestContext.Current.CancellationToken); + } while (sw.Elapsed < budget); + return last; + } + + /// + /// P3: after the device drops (two failed calls in a row, socket torn down, reconnect refused) the + /// transport must reconnect ON ITS OWN once the device is back — before this fix every later call + /// failed fast with "Transport not connected" until the 90 s comms-loss watchdog re-initialised the + /// driver. + /// + [Fact] + public async Task Driver_reads_again_after_the_device_drops_and_returns_without_reinit() + { + SkipUnlessStandardLive(); + await using var relay = new TcpRelay(sim.Host, sim.Port); + var tag = new ModbusTagDefinition("HR5", ModbusRegion.HoldingRegisters, 5, ModbusDataType.UInt16, Writable: false); + await using var driver = new ModbusDriver(Options(relay, tag), "modbus-p3"); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var good = await driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken); + good[0].StatusCode.ShouldBe(StatusGood); + good[0].Value.ShouldBe((ushort)5); + + relay.Cut(); + for (var i = 0; i < 2; i++) + { + var bad = await driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken); + bad[0].StatusCode.ShouldNotBe(StatusGood); + } + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + + relay.Restore(); + var recovered = await WaitFor( + () => driver.ReadAsync(["HR5"], TestContext.Current.CancellationToken), + r => r[0].StatusCode == StatusGood, + TimeSpan.FromSeconds(20)); + recovered[0].StatusCode.ShouldBe(StatusGood, "the transport must reconnect by itself once the device is back"); + recovered[0].Value.ShouldBe((ushort)5); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/TcpRelay.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/TcpRelay.cs new file mode 100644 index 00000000..416d0e72 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/TcpRelay.cs @@ -0,0 +1,101 @@ +using System.Net; +using System.Net.Sockets; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests; + +/// +/// Loopback TCP relay in front of the real Modbus fixture, so a test can make the device "disappear" +/// (: every relayed connection is closed and new connects are refused) and "come back" +/// (: listen again on the SAME port) without touching the shared container. +/// +internal sealed class TcpRelay : IAsyncDisposable +{ + private readonly string _upstreamHost; + private readonly int _upstreamPort; + private readonly object _lock = new(); + private readonly List _live = []; + private TcpListener? _listener; + private CancellationTokenSource _stop = new(); + private int _accepted; + + public TcpRelay(string upstreamHost, int upstreamPort) + { + _upstreamHost = upstreamHost; + _upstreamPort = upstreamPort; + var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + Port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + Restore(); + } + + /// Gets the loopback port clients dial. + public int Port { get; } + + /// Gets the number of client connections the relay has accepted. + public int AcceptedConnections => Volatile.Read(ref _accepted); + + /// Starts (or resumes) relaying on . + public void Restore() + { + _stop = new CancellationTokenSource(); + var listener = new TcpListener(IPAddress.Loopback, Port); + listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + listener.Start(); + _listener = listener; + var token = _stop.Token; + _ = Task.Run(() => AcceptLoopAsync(listener, token)); + } + + /// Closes every relayed connection and refuses new ones until . + public void Cut() + { + try { _stop.Cancel(); } catch { } + try { _listener?.Stop(); } catch { } + _listener = null; + lock (_lock) + { + foreach (var c in _live) { try { c.Close(); } catch { } } + _live.Clear(); + } + } + + private async Task AcceptLoopAsync(TcpListener listener, CancellationToken token) + { + while (!token.IsCancellationRequested) + { + TcpClient downstream; + try { downstream = await listener.AcceptTcpClientAsync(token); } + catch { return; } + Interlocked.Increment(ref _accepted); + _ = Task.Run(() => RelayAsync(downstream, token)); + } + } + + private async Task RelayAsync(TcpClient downstream, CancellationToken token) + { + var upstream = new TcpClient(AddressFamily.InterNetwork); + lock (_lock) { _live.Add(downstream); _live.Add(upstream); } + try + { + var addr = (await Dns.GetHostAddressesAsync(_upstreamHost, token)) + .First(a => a.AddressFamily == AddressFamily.InterNetwork); + await upstream.ConnectAsync(addr, _upstreamPort, token); + var a = downstream.GetStream(); + var b = upstream.GetStream(); + await Task.WhenAny(a.CopyToAsync(b, token), b.CopyToAsync(a, token)); + } + catch { /* cut or upstream gone */ } + finally + { + try { downstream.Close(); } catch { } + try { upstream.Close(); } catch { } + } + } + + public ValueTask DisposeAsync() + { + Cut(); + return ValueTask.CompletedTask; + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTransportSelfHealTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTransportSelfHealTests.cs new file mode 100644 index 00000000..22e3a93b --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTransportSelfHealTests.cs @@ -0,0 +1,139 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Transport self-heal (review 07 P3), single in-gate connect attempt + failed-client disposal (L3) +/// and the response-PDU length check (H9). +/// +[Trait("Category", "Unit")] +public sealed class ModbusTransportSelfHealTests +{ + private static readonly byte[] ReadHr0 = [0x03, 0x00, 0x00, 0x00, 0x01]; + + /// + /// P3: two lost replies in a row leave the socket torn down. The NEXT call must reconnect on its own + /// instead of failing fast with "Transport not connected" until the 90 s comms-loss watchdog. + /// + [Fact] + public async Task After_two_consecutive_timeouts_the_next_call_reconnects() + { + await using var server = new ScriptedModbusTcpServer + { + Script = i => i < 2 ? ScriptedModbusTcpServer.Action.Stall : ScriptedModbusTcpServer.Action.Answer, + }; + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromMilliseconds(500)); + await transport.ConnectAsync(TestContext.Current.CancellationToken); + + // First call: stall → teardown → reconnect → resend → stall again → propagates. + await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + + // Second call: the socket is gone; the transport must dial again and succeed. + var resp = await transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken); + resp[0].ShouldBe((byte)0x03); + server.AcceptedConnections.ShouldBe(3); + } + + /// + /// P3 throttle: after a FAILED connect, calls inside the backoff window fail fast with + /// and make no connect attempt; once the window has + /// passed the next call dials again. Positive control: the device is back before the throttled call, + /// so an un-throttled transport would have reconnected. + /// + [Fact] + public async Task Failed_connect_is_throttled_then_retried_after_the_window() + { + await using var server = new ScriptedModbusTcpServer(); + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromSeconds(2)); + var now = 1_000_000L; + transport.Lifecycle.NowMs = () => now; + await transport.ConnectAsync(TestContext.Current.CancellationToken); + (await transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken))[0].ShouldBe((byte)0x03); + + server.Stop(); + // Dead socket → EOF → teardown → ONE reconnect attempt → refused. + var first = await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + ModbusSocketLifecycle.IsSocketLevelFailure(first).ShouldBeTrue(); + + server.Start(); + var acceptedBefore = server.AcceptedConnections; + await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + server.AcceptedConnections.ShouldBe(acceptedBefore, "a throttled call must not dial"); + + now += 60_000; // past any window (MaxDelay default 30 s) + (await transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken))[0].ShouldBe((byte)0x03); + server.AcceptedConnections.ShouldBe(acceptedBefore + 1); + } + + /// + /// L3: the reconnect never sleeps inside the transport gate. With a 30 s InitialDelay the old + /// ConnectWithBackoffAsync held the gate for 30 s before even dialling; now one attempt is made + /// and the call returns at once. + /// + [Fact] + public async Task Reconnect_after_drop_does_not_sleep_inside_the_gate() + { + await using var server = new ScriptedModbusTcpServer(); + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromSeconds(2), + reconnect: new ModbusReconnectOptions { InitialDelay = TimeSpan.FromSeconds(30) }); + await transport.ConnectAsync(TestContext.Current.CancellationToken); + server.Stop(); + + var call = transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken); + var ex = await Should.ThrowAsync(() => call.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)); + ex.ShouldNotBeOfType("the send must not wait out the 30 s backoff inside the gate"); + } + + /// H9: an MBAP frame carrying no PDU is a truncated-frame desync, not an IndexOutOfRangeException. + [Fact] + public async Task Empty_response_pdu_is_a_truncated_frame_desync() + { + await using var server = new ScriptedModbusTcpServer + { + RawReply = (_, header, _) => ScriptedModbusTcpServer.Adu(header, []), + }; + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromSeconds(2), autoReconnect: false); + await transport.ConnectAsync(TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + ex.Reason.ShouldBe(ModbusTransportDesyncException.DesyncReason.TruncatedFrame); + } + + /// H9: an exception frame missing its exception code is a truncated-frame desync. + [Fact] + public async Task Exception_frame_without_code_is_a_truncated_frame_desync() + { + await using var server = new ScriptedModbusTcpServer + { + RawReply = (_, header, _) => ScriptedModbusTcpServer.Adu(header, [0x83]), + }; + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromSeconds(2), autoReconnect: false); + await transport.ConnectAsync(TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + ex.Reason.ShouldBe(ModbusTransportDesyncException.DesyncReason.TruncatedFrame); + } + + /// Positive control for the two H9 tests: a well-formed exception frame still surfaces as a + /// carrying its code. + [Fact] + public async Task Well_formed_exception_frame_still_surfaces_as_ModbusException() + { + await using var server = new ScriptedModbusTcpServer + { + RawReply = (_, header, _) => ScriptedModbusTcpServer.Adu(header, [0x83, 0x02]), + }; + await using var transport = new ModbusTcpTransport("127.0.0.1", server.Port, TimeSpan.FromSeconds(2), autoReconnect: false); + await transport.ConnectAsync(TestContext.Current.CancellationToken); + + var ex = await Should.ThrowAsync( + () => transport.SendAsync(1, ReadHr0, TestContext.Current.CancellationToken)); + ex.ExceptionCode.ShouldBe((byte)0x02); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedModbusTcpServer.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedModbusTcpServer.cs new file mode 100644 index 00000000..1fa6a505 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedModbusTcpServer.cs @@ -0,0 +1,160 @@ +using System.Net; +using System.Net.Sockets; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Loopback Modbus-TCP stub whose reply to each request is scripted by the test, and whose listener can +/// be stopped (connection refused) and restarted on the SAME port. Used by the transport self-heal tests +/// that need a device which disappears and comes back. +/// +internal sealed class ScriptedModbusTcpServer : IAsyncDisposable +{ + /// What the server does with one request. + public enum Action + { + /// Answer with an FC03-shaped response of zeroed registers. + Answer, + /// Read the request and never answer it (the client's per-op timeout fires). + Stall, + /// Close the socket without answering. + Drop, + } + + private TcpListener? _listener; + private CancellationTokenSource _stop = new(); + private int _requests; + private int _accepted; + + /// Optional scripted reply. Receives the 0-based request index and the request PDU; returns + /// the raw bytes to write AFTER the MBAP TxId (i.e. the full ADU is built by the test), or null to fall + /// through to . + public Func? RawReply { get; set; } + + /// Per-request action; defaults to . + public Func Script { get; set; } = _ => Action.Answer; + + /// Gets the port the server listens on (stable across /). + public int Port { get; } + + /// Gets the number of accepted TCP connections. + public int AcceptedConnections => Volatile.Read(ref _accepted); + + /// Gets the number of requests received. + public int Requests => Volatile.Read(ref _requests); + + public ScriptedModbusTcpServer() + { + var probe = new TcpListener(IPAddress.Loopback, 0); + probe.Start(); + Port = ((IPEndPoint)probe.LocalEndpoint).Port; + probe.Stop(); + Start(); + } + + /// (Re)starts listening on . + public void Start() + { + _stop = new CancellationTokenSource(); + _listener = new TcpListener(IPAddress.Loopback, Port); + _listener.Server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _listener.Start(); + var listener = _listener; + var token = _stop.Token; + _ = Task.Run(() => AcceptLoopAsync(listener, token)); + } + + /// Stops listening and closes every live connection — new connects are refused. + public void Stop() + { + try { _stop.Cancel(); } catch { } + try { _listener?.Stop(); } catch { } + _listener = null; + } + + private async Task AcceptLoopAsync(TcpListener listener, CancellationToken token) + { + while (!token.IsCancellationRequested) + { + TcpClient client; + try { client = await listener.AcceptTcpClientAsync(token); } + catch { return; } + Interlocked.Increment(ref _accepted); + _ = Task.Run(() => ServeAsync(client, token)); + } + } + + private async Task ServeAsync(TcpClient client, CancellationToken token) + { + using var _ = client; + using var reg = token.Register(() => { try { client.Close(); } catch { } }); + try + { + var stream = client.GetStream(); + while (!token.IsCancellationRequested) + { + var header = new byte[7]; + if (!await ReadExactly(stream, header, token)) return; + var len = (ushort)((header[4] << 8) | header[5]); + var pdu = new byte[len - 1]; + if (!await ReadExactly(stream, pdu, token)) return; + var idx = Interlocked.Increment(ref _requests) - 1; + + var raw = RawReply?.Invoke(idx, header, pdu); + if (raw is not null) + { + await stream.WriteAsync(raw, token); + await stream.FlushAsync(token); + continue; + } + + switch (Script(idx)) + { + case Action.Stall: + continue; + case Action.Drop: + client.Client.Shutdown(SocketShutdown.Both); + return; + } + + var qty = (ushort)((pdu[3] << 8) | pdu[4]); + var respPdu = new byte[2 + qty * 2]; + respPdu[0] = pdu[0]; + respPdu[1] = (byte)(qty * 2); + await stream.WriteAsync(Adu(header, respPdu), token); + await stream.FlushAsync(token); + } + } + catch { /* connection torn down */ } + } + + /// Builds a reply ADU echoing the request's TxId/unit around . + public static byte[] Adu(byte[] requestHeader, byte[] respPdu) + { + var respLen = (ushort)(1 + respPdu.Length); + var adu = new byte[7 + respPdu.Length]; + adu[0] = requestHeader[0]; adu[1] = requestHeader[1]; + adu[4] = (byte)(respLen >> 8); adu[5] = (byte)(respLen & 0xFF); + adu[6] = requestHeader[6]; + Buffer.BlockCopy(respPdu, 0, adu, 7, respPdu.Length); + return adu; + } + + private static async Task ReadExactly(Stream s, byte[] buf, CancellationToken token) + { + var read = 0; + while (read < buf.Length) + { + var n = await s.ReadAsync(buf.AsMemory(read), token); + if (n == 0) return false; + read += n; + } + return true; + } + + public ValueTask DisposeAsync() + { + Stop(); + return ValueTask.CompletedTask; + } +} From ae20519e9d676c9ca61ecd34eeb5811cc79c34c2 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:05:25 -0400 Subject: [PATCH 02/34] fix(modbus): deadband baseline on (value, status) so the Good recovery publishes H2: a Bad/null publish drops the deadband baseline and any status change publishes, so a device that returns holding the identical value is no longer swallowed by |new - old| < deadband (the #527 value-only-dedup trap). Integration case drives a real drop/restore through a loopback relay. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriver.cs | 44 ++++++++++----- .../CommsRecoveryTests.cs | 43 +++++++++++++++ .../ModbusDeadbandRecoveryTests.cs | 53 +++++++++++++++++++ 3 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDeadbandRecoveryTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index 19e90946..fc85ddde 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -49,10 +49,10 @@ public sealed class ModbusDriver // authored _tagsByRawPath table (v3: the reference is always a RawPath; a miss is a miss). private readonly EquipmentTagRefResolver _resolver; - // Last-published value per tag, keyed by FullReference. Used by ShouldPublish to apply - // the deadband filter. Stored as object so all numeric types share one map; the comparison - // does a typed cast inside. - private readonly ConcurrentDictionary _lastPublishedByRef = new(StringComparer.OrdinalIgnoreCase); + // Last-published (value, status) per tag, keyed by FullReference. Used by ShouldPublish to apply + // the deadband filter. The value is stored as object so all numeric types share one map; the + // comparison does a typed cast inside. The status is part of the baseline — see ShouldPublish. + private readonly ConcurrentDictionary _lastPublishedByRef = new(StringComparer.OrdinalIgnoreCase); // Per-cycle buffer of the items this driver actually PUBLISHED (i.e. that passed ShouldPublish), // keyed by subscription handle. The poll engine collects its own batch list unconditionally — it @@ -233,31 +233,51 @@ public sealed class ModbusDriver /// Format a per-slave host string. Multi-slave deployments distinguish breakers by this string. private string BuildSlaveHostName(byte unitId) => $"{_options.Host}:{_options.Port}/unit{unitId}"; - private bool ShouldPublish(string tagRef, DataValueSnapshot snapshot) + /// + /// Deadband filter for the subscribe path. The baseline is keyed on (value, status), never + /// value alone: a Bad/null publish drops the baseline, and a status change always publishes. A + /// device that comes back after a comms drop usually comes back holding the SAME reading, so the + /// old value-only comparison (|10 − 10| < deadband) swallowed the Good recovery and left + /// the node Bad until the value drifted past the deadband — the #527 trap. + /// + /// The subscribed reference. + /// The snapshot the poll engine is about to raise. + /// when the snapshot must be published. + internal bool ShouldPublish(string tagRef, DataValueSnapshot snapshot) { if (!_resolver.TryResolve(tagRef, out var tag) || tag.Deadband is null) return true; - if (snapshot.Value is null) return true; // Deadband only applies to numeric scalar types — array / Bool / String publishes // unconditionally. Easier to special-case skip than to enumerate the supported types. if (tag.ArrayCount.HasValue || tag.DataType is ModbusDataType.Bool or ModbusDataType.BitInRegister or ModbusDataType.String) return true; - if (!_lastPublishedByRef.TryGetValue(tagRef, out var prev)) + if (snapshot.Value is null || StatusCodeIsBad(snapshot.StatusCode)) { - // First sample passes through unconditionally — the threshold can't be evaluated - // without a baseline. The publish lands and seeds the comparison. - _lastPublishedByRef[tagRef] = snapshot.Value; + // A Bad publish always passes AND drops the baseline, so the next Good sample is + // compared against nothing and publishes — whatever value it carries. + _lastPublishedByRef.TryRemove(tagRef, out _); + return true; + } + + if (!_lastPublishedByRef.TryGetValue(tagRef, out var prev) || prev.Status != snapshot.StatusCode) + { + // First sample, or a quality transition (Good ↔ Uncertain): passes unconditionally and + // (re)seeds the comparison. The threshold cannot be evaluated across a status change. + _lastPublishedByRef[tagRef] = (snapshot.Value, snapshot.StatusCode); return true; } var newD = Convert.ToDouble(snapshot.Value); - var oldD = Convert.ToDouble(prev); + var oldD = Convert.ToDouble(prev.Value); if (Math.Abs(newD - oldD) < tag.Deadband.Value) return false; - _lastPublishedByRef[tagRef] = snapshot.Value; + _lastPublishedByRef[tagRef] = (snapshot.Value, snapshot.StatusCode); return true; } + /// OPC UA severity bits 31..30 = 10 ⇒ Bad. + private static bool StatusCodeIsBad(uint statusCode) => (statusCode & 0xC0000000u) == 0x80000000u; + /// public string DriverInstanceId => _driverInstanceId; /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs index 37192239..857ce3b7 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs @@ -85,4 +85,47 @@ public sealed class CommsRecoveryTests(ModbusSimulatorFixture sim) recovered[0].Value.ShouldBe((ushort)5); driver.GetHealth().State.ShouldBe(DriverState.Healthy); } + + /// + /// H2: a deadbanded tag whose device returns holding the SAME value must publish the Good recovery. + /// HR[5] is a constant 5 in the standard profile, so the recovery value is identical by construction. + /// Positive control: the Bad publish during the outage is observed first. + /// + [Fact] + public async Task Deadbanded_subscription_publishes_the_Good_recovery_with_an_identical_value() + { + SkipUnlessStandardLive(); + await using var relay = new TcpRelay(sim.Host, sim.Port); + var tag = new ModbusTagDefinition("HR5", ModbusRegion.HoldingRegisters, 5, ModbusDataType.UInt16, + Writable: false, Deadband: 10.0); + await using var driver = new ModbusDriver(Options(relay, tag), "modbus-h2"); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var published = new System.Collections.Concurrent.ConcurrentQueue(); + driver.OnDataChange += (_, e) => published.Enqueue(e.Snapshot); + var handle = await driver.SubscribeAsync(["HR5"], TimeSpan.FromMilliseconds(200), TestContext.Current.CancellationToken); + try + { + (await WaitFor(() => Task.FromResult(published.ToArray()), + p => p.Any(s => s.StatusCode == StatusGood), TimeSpan.FromSeconds(10))) + .ShouldContain(s => s.StatusCode == StatusGood, "initial Good publish"); + + relay.Cut(); + (await WaitFor(() => Task.FromResult(published.ToArray()), + p => p.Any(s => s.StatusCode != StatusGood), TimeSpan.FromSeconds(20))) + .ShouldContain(s => s.StatusCode != StatusGood, "the outage must publish Bad (positive control)"); + + var badCount = published.Count; + relay.Restore(); + var after = await WaitFor(() => Task.FromResult(published.Skip(badCount).ToArray()), + p => p.Any(s => s.StatusCode == StatusGood), TimeSpan.FromSeconds(30)); + var recovery = after.FirstOrDefault(s => s.StatusCode == StatusGood); + recovery.ShouldNotBeNull("the Good recovery with the same value must not be swallowed by the deadband"); + recovery.Value.ShouldBe((ushort)5); + } + finally + { + await driver.UnsubscribeAsync(handle, TestContext.Current.CancellationToken); + } + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDeadbandRecoveryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDeadbandRecoveryTests.cs new file mode 100644 index 00000000..43275c15 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDeadbandRecoveryTests.cs @@ -0,0 +1,53 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Review 07 H2: the deadband baseline is keyed on (value, status). A Good-after-Bad recovery carrying +/// the IDENTICAL value must publish — the value-only dedup left the node Bad until the value drifted. +/// +[Trait("Category", "Unit")] +public sealed class ModbusDeadbandRecoveryTests +{ + private const uint Good = 0u; + private const uint Uncertain = 0x40000000u; + private const uint BadComm = 0x80050000u; + + private static async Task NewDriverAsync() + { + var tag = new ModbusTagDefinition("Temp", ModbusRegion.HoldingRegisters, 0, ModbusDataType.Int16, Deadband: 5.0); + var opts = new ModbusDriverOptions + { + Host = "f", RawTags = ModbusRawTags.Entries([tag]), Probe = new ModbusProbeOptions { Enabled = false }, + }; + var drv = new ModbusDriver(opts, "m1", _ => new ModbusDriverTests.FakeTransport()); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + return drv; + } + + private static DataValueSnapshot Snap(object? v, uint status) => new(v, status, DateTime.UtcNow, DateTime.UtcNow); + + [Fact] + public async Task Good_after_Bad_with_the_same_value_publishes() + { + await using var drv = await NewDriverAsync(); + drv.ShouldPublish("Temp", Snap((short)10, Good)).ShouldBeTrue(); // baseline + drv.ShouldPublish("Temp", Snap(null, BadComm)).ShouldBeTrue(); // comms drop + drv.ShouldPublish("Temp", Snap((short)10, Good)).ShouldBeTrue("the recovery publish must not be swallowed"); + // Positive control: the deadband is still active after the recovery re-seeded the baseline. + drv.ShouldPublish("Temp", Snap((short)11, Good)).ShouldBeFalse(); + drv.ShouldPublish("Temp", Snap((short)16, Good)).ShouldBeTrue(); + } + + [Fact] + public async Task A_status_change_with_the_same_value_publishes() + { + await using var drv = await NewDriverAsync(); + drv.ShouldPublish("Temp", Snap((short)10, Good)).ShouldBeTrue(); + drv.ShouldPublish("Temp", Snap((short)10, Uncertain)).ShouldBeTrue("a quality change must publish"); + drv.ShouldPublish("Temp", Snap((short)12, Uncertain)).ShouldBeFalse("within deadband at the same quality"); + drv.ShouldPublish("Temp", Snap((short)10, Good)).ShouldBeTrue("back to Good must publish"); + } +} From 620ecc41dddb6386e73c43b971a443da680a5ebc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:15:20 -0400 Subject: [PATCH 03/34] fix(modbus): refusals are per-tag Bad, batch health, generation-stamped lifecycle, write status mapping H4: exception 01/02/03 (and undecodable data) are device refusals - per-tag Bad with a once-per-tag Warning, no Degraded, and they advance LastSuccessfulRead (the device answered), so an all-refused driver no longer trips the 90 s watchdog. 04/05/06 still degrade (answered); 0A/0B degrade without advancing. Health is one worst-wins write per batch instead of last-writer-wins per tag. A comms failure on one unit fails the rest of that unit's batch fast; an unreachable endpoint fails the whole batch fast. H7: write transport failures map to BadCommunicationError, value overflow and array length mismatch to BadOutOfRange, conversion failures to BadTypeMismatch; writes report write-side health (a good write or a refusal clears a write-side Degraded; a good read does not). Lifecycle: every health write and the transport/table publish carry a generation; a superseded init releases its transport and publishes nothing, a read straddling a re-init cannot degrade the new transport, and the shutdown's final write cannot clobber a newer init. The tag table is build-and-swapped (re-init no longer merges into the old table). Teardown joins the probe/re-probe loops (bounded); a probe answer advances LastSuccessfulRead and an exception PDU from the device counts as Running. Transports latch dispose atomically and never reconnect after it. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDecodeException.cs | 17 + .../ModbusDriver.cs | 611 ++++++++++++++---- .../ModbusRtuOverTcpTransport.cs | 9 +- .../ModbusSocketLifecycle.cs | 19 +- .../ModbusTcpTransport.cs | 9 +- .../ModbusWriteRefusedException.cs | 21 + .../ModbusHealthContractTests.cs | 341 ++++++++++ .../ModbusLoggerInjectionTests.cs | 10 +- .../ScriptedTransport.cs | 51 ++ 9 files changed, 939 insertions(+), 149 deletions(-) create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDecodeException.cs create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusWriteRefusedException.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDecodeException.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDecodeException.cs new file mode 100644 index 00000000..06d9a639 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDecodeException.cs @@ -0,0 +1,17 @@ +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus; + +/// +/// The device answered with well-framed data the tag's codec cannot represent (a non-BCD nibble in a +/// BCD register, for example). The device is reachable, so this is a per-tag +/// BadDecodingError with a once-per-tag Warning — never a comms fault, never a health write. +/// +public sealed class ModbusDecodeException : Exception +{ + /// Initializes a new instance of the class. + /// A human-readable description. + /// The codec's own exception. + public ModbusDecodeException(string message, Exception? inner = null) + : base(message, inner) + { + } +} diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index fc85ddde..648e4c99 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -43,7 +43,33 @@ public sealed class ModbusDriver // v3 authored-tag table, keyed by RawPath (the tag identity + driver wire reference). RawPath is // case-sensitive ordinal. Built in InitializeAsync from _options.RawTags via the pure mapper. - private readonly Dictionary _tagsByRawPath = new(StringComparer.Ordinal); + // BUILD-AND-SWAP: a fresh dictionary is built off to the side and published by reference + // (Volatile.Write, under _stateLock with the generation check); it is never mutated or .Clear()ed + // in place, because retained poll loops read it concurrently across a re-initialize. + private Dictionary _tagsByRawPath = new(StringComparer.Ordinal); + + /// + /// Lifecycle generation. Every InitializeAsync / ShutdownAsync bumps it under + /// ; every health write and every table/transport publish carries the + /// generation it was computed under and is DROPPED once the generation has moved on — so a + /// superseded (abandoned) initialize can never publish over a newer one, a read that straddled a + /// re-initialize cannot write health about the new transport, and a shutdown's final health + /// write cannot clobber a newer init. + /// + private int _generation; + + /// Guards , the transport/table publish and the health fields. + private readonly object _stateLock = new(); + + // Health is composed from two independent sides so a good write clears only a WRITE-side + // Degraded (and a good read only a read-side one). LastSuccessfulRead advances on any evidence the + // device answered — a good read, a good write, a refusal, or a probe answer. + private string? _readFault; + private string? _writeFault; + + // Once-per-tag Warning for deterministic device refusals (exception 01/02/03, undecodable data). + // Replaced (never cleared in place) on every table publish. + private ConcurrentDictionary _refusalWarned = new(StringComparer.Ordinal); // Resolves a read/write/subscribe RawPath reference to its tag definition via a single hit on the // authored _tagsByRawPath table (v3: the reference is always a RawPath; a miss is a miss). @@ -87,6 +113,11 @@ public sealed class ModbusDriver private IModbusTransport? _transport; private CancellationTokenSource? _probeCts; private CancellationTokenSource? _reprobeCts; + private Task? _probeTask; + private Task? _reprobeTask; + + /// Bounded join on the probe / re-probe loops at teardown — they honour their token, so this is a backstop. + private static readonly TimeSpan BackgroundLoopJoinTimeout = TimeSpan.FromSeconds(2); /// /// Guards / / @@ -105,7 +136,7 @@ public sealed class ModbusDriver /// (a connect that never returns) must not turn shutdown or dispose into a hang — past the /// timeout they tear down anyway, which is the lesser evil of the two. /// - private static readonly TimeSpan LifecycleGateTimeout = TimeSpan.FromSeconds(10); + internal TimeSpan LifecycleGateTimeout { get; set; } = TimeSpan.FromSeconds(10); private DriverHealth _health = new(DriverState.Unknown, null, null); @@ -152,7 +183,7 @@ public sealed class ModbusDriver _driverInstanceId = driverInstanceId; _logger = logger ?? NullLogger.Instance; _resolver = new EquipmentTagRefResolver( - r => _tagsByRawPath.TryGetValue(r, out var t) ? t : null); + r => Volatile.Read(ref _tagsByRawPath).TryGetValue(r, out var t) ? t : null); _transportFactory = transportFactory ?? (o => ModbusTransportFactory.Create(o)); _poll = new PollGroupEngine( @@ -208,9 +239,151 @@ public sealed class ModbusDriver internal void HandlePollError(Exception ex) { _logger.LogWarning(ex, "Modbus poll reader failed. Driver={DriverInstanceId}", _driverInstanceId); - var current = ReadHealth(); - if (current.State != DriverState.Faulted) - WriteHealth(new DriverHealth(DriverState.Degraded, current.LastSuccessfulRead, ex.Message)); + lock (_stateLock) + { + // Lifecycle-owned states are left alone: Faulted is a stronger, config-level signal, and an + // Initializing/Unknown driver has no transport for a poll to have failed against — the + // failure is the re-initialize window itself, which the lifecycle reports on its own. + var current = _health; + if (current.State is DriverState.Faulted or DriverState.Initializing or DriverState.Unknown) return; + _readFault = ex.Message; + PublishComposedHealthLocked(current.LastSuccessfulRead); + } + } + + // ---- health composition ---- + + /// Which half of the driver a batch outcome came from. + private enum HealthSide { Read, Write } + + /// + /// Per-batch accounting: did the device answer anything, did a comms fault happen, and which units + /// (or the whole endpoint) must the rest of this batch fail fast against. + /// + private sealed class BatchOutcome + { + public bool Answered; + public string? CommsFault; + public bool EndpointDown; + private HashSet? _failedUnits; + + public void FailUnit(byte unit) => (_failedUnits ??= []).Add(unit); + + public bool IsFailedFast(byte unit) => EndpointDown || (_failedUnits?.Contains(unit) ?? false); + } + + /// + /// Applies one batch's outcome to the driver health, under , and only when + /// is still current. A batch that touched no device (every reference + /// unknown, every write refused before the wire) changes nothing. + /// + private void ReportOutcome(int generation, HealthSide side, BatchOutcome batch, DateTime now) + { + if (!batch.Answered && batch.CommsFault is null) return; + lock (_stateLock) + { + if (generation != _generation) return; + var current = _health; + if (current.State is DriverState.Faulted or DriverState.Initializing or DriverState.Unknown) return; + if (side == HealthSide.Read) _readFault = batch.CommsFault; + else _writeFault = batch.CommsFault; + PublishComposedHealthLocked(batch.Answered ? now : current.LastSuccessfulRead); + } + } + + /// Writes Healthy/Degraded from the two fault sides. Caller holds . + private void PublishComposedHealthLocked(DateTime? lastSuccessfulRead) + { + var fault = _readFault ?? _writeFault; + WriteHealth(fault is null + ? new DriverHealth(DriverState.Healthy, lastSuccessfulRead, null) + : new DriverHealth(DriverState.Degraded, lastSuccessfulRead, fault)); + } + + /// + /// A probe answer proves the device is reachable: advance LastSuccessfulRead without + /// touching either fault side (it says nothing about the tags). + /// + private void NoteDeviceAnswered(int generation) + { + lock (_stateLock) + { + if (generation != _generation) return; + var current = _health; + if (current.State is not (DriverState.Healthy or DriverState.Degraded)) return; + WriteHealth(current with { LastSuccessfulRead = DateTime.UtcNow }); + } + } + + /// Lifecycle health write, dropped when has been superseded. + private void WriteLifecycleHealth(int generation, DriverHealth health) + { + lock (_stateLock) + { + if (generation != _generation) return; + _readFault = null; + _writeFault = null; + WriteHealth(health); + } + } + + /// + /// Classifies one failed read/write against the health contract and returns the per-tag status: + /// + /// Exception 01/02/03 — a deterministic REFUSAL: the device answered; per-tag Bad, a + /// once-per-tag Warning, no Degraded (it used to latch Degraded and, if every tag refused, + /// trip the 90 s watchdog against a reachable device). + /// Exception 04/05/06 — the device answered but is not serving: Degraded, answered. + /// Exception 0A/0B (and unknown codes) — the gateway answered for a target that did not: + /// Degraded, not answered; 0B fails the rest of that unit's batch fast. + /// Undecodable data — the device answered with a value we cannot represent: per-tag + /// BadDecodingError, once-per-tag Warning, no Degraded. + /// No socket / throttled reconnect — the endpoint is down: Degraded, the rest of the + /// batch fails fast. + /// Any other transport failure (timeout, desync, EOF) — Degraded; the rest of that + /// unit's batch fails fast instead of serialising N timeouts. + /// + /// + private uint ClassifyFailure(Exception ex, byte unit, string tagRef, BatchOutcome batch, bool isWrite) + { + switch (ex) + { + case ModbusException mex when IsRefusalCode(mex.ExceptionCode): + batch.Answered = true; + WarnRefusalOnce(tagRef, isWrite, $"exception 0x{mex.ExceptionCode:X2} (fc 0x{mex.FunctionCode:X2})"); + return MapModbusExceptionToStatus(mex.ExceptionCode); + case ModbusException mex when mex.ExceptionCode is 0x04 or 0x05 or 0x06: + batch.Answered = true; + batch.CommsFault ??= mex.Message; + return MapModbusExceptionToStatus(mex.ExceptionCode); + case ModbusException mex: + batch.CommsFault ??= mex.Message; + if (mex.ExceptionCode == 0x0B) batch.FailUnit(unit); + return MapModbusExceptionToStatus(mex.ExceptionCode); + case ModbusDecodeException dex: + batch.Answered = true; + WarnRefusalOnce(tagRef, isWrite, dex.Message); + return StatusBadDecodingError; + case ModbusTransportUnavailableException or ObjectDisposedException or System.Net.Sockets.SocketException: + batch.CommsFault ??= ex.Message; + batch.EndpointDown = true; + return StatusBadCommunicationError; + default: + batch.CommsFault ??= ex.Message; + batch.FailUnit(unit); + return StatusBadCommunicationError; + } + } + + /// Exception 01 Illegal Function / 02 Illegal Data Address / 03 Illegal Data Value — the device refused the request. + internal static bool IsRefusalCode(byte exceptionCode) => exceptionCode is 0x01 or 0x02 or 0x03; + + private void WarnRefusalOnce(string tagRef, bool isWrite, string detail) + { + if (!Volatile.Read(ref _refusalWarned).TryAdd((isWrite ? "w:" : "r:") + tagRef, 0)) return; + _logger.LogWarning( + "Modbus device refused {Operation} of tag {RawPath}: {Detail}. The tag reads/writes Bad; driver health is unaffected (the device answered). Driver={DriverInstanceId}", + isWrite ? "a write" : "a read", tagRef, detail, _driverInstanceId); } // @@ -310,11 +483,26 @@ public sealed class ModbusDriver /// The gate-held body of . private async Task InitializeCoreAsync(string driverConfigJson, CancellationToken cancellationToken) { - WriteHealth(new DriverHealth(DriverState.Initializing, null, null)); + // Claim a new generation and take the previous transport out of service atomically: any read or + // write that captured the old generation can no longer write health, and nothing can capture the + // old transport from here on. + int generation; + IModbusTransport? previous; + lock (_stateLock) + { + generation = ++_generation; + previous = _transport; + _transport = null; + _readFault = null; + _writeFault = null; + WriteHealth(new DriverHealth(DriverState.Initializing, null, null)); + } + + IModbusTransport? transport = null; try { // Release whatever a PREVIOUS initialize published before building the replacements. All - // three fields assigned below (_transport, _probeCts, _reprobeCts) were simply overwritten, + // fields assigned below (_transport, _probeCts, _reprobeCts) were simply overwritten, // so a second initialize on a live driver stranded a connected transport plus two running // background loops that no later dispose could reach. The IDriver contract lets a // caller re-initialise a live driver without ShutdownAsync, and DriverInstanceActor's retry tick @@ -323,20 +511,9 @@ public sealed class ModbusDriver // every ACTIVE SUBSCRIPTION — and nothing re-subscribes them afterwards, because a // mid-life re-initialize is not a shutdown the OPC UA layer knows to re-drive. A caller // that genuinely wants the subscriptions gone calls ShutdownAsync. A first initialize - // sees three nulls, so this block is a no-op on the initial call. - try { _probeCts?.Cancel(); } catch { } - _probeCts?.Dispose(); - _probeCts = null; - - try { _reprobeCts?.Cancel(); } catch { } - _reprobeCts?.Dispose(); - _reprobeCts = null; - - if (_transport is not null) - { - await _transport.DisposeAsync().ConfigureAwait(false); - _transport = null; - } + // sees nulls, so this block is a no-op on the initial call. + await StopBackgroundLoopsAsync().ConfigureAwait(false); + if (previous is not null) await previous.DisposeAsync().ConfigureAwait(false); // re-parse the supplied DriverConfig so a config change delivered through the IDriver // contract is honoured. Without this the driver keeps serving the options it was CONSTRUCTED @@ -345,17 +522,19 @@ public sealed class ModbusDriver if (HasConfigBody(driverConfigJson)) _options = ModbusDriverFactoryExtensions.ParseOptions(_driverInstanceId, driverConfigJson); - _transport = _transportFactory(_options); - await _transport.ConnectAsync(cancellationToken).ConfigureAwait(false); - // Build the RawPath → definition table from the authored raw tags. Each entry's TagConfig - // blob is mapped by the pure factory; the entry's WriteIdempotent flag is threaded onto the - // def (it lives on the RawTagEntry, not inside the blob). A mapper miss is skipped (logged), - // never thrown — a bad tag must not fail the whole driver init. + transport = _transportFactory(_options); + await transport.ConnectAsync(cancellationToken).ConfigureAwait(false); + + // Build the RawPath → definition table OFF TO THE SIDE from the authored raw tags. Each + // entry's TagConfig blob is mapped by the pure factory; the entry's WriteIdempotent flag is + // threaded onto the def (it lives on the RawTagEntry, not inside the blob). A mapper miss is + // skipped (logged), never thrown — a bad tag must not fail the whole driver init. + var table = new Dictionary(StringComparer.Ordinal); _skippedTags.BeginRebuild(_options.RawTags.Count); foreach (var entry in _options.RawTags) { if (ModbusTagDefinitionFactory.FromTagConfig(entry.TagConfig, entry.RawPath, out var def)) - _tagsByRawPath[entry.RawPath] = def with { WriteIdempotent = entry.WriteIdempotent }; + table[entry.RawPath] = def with { WriteIdempotent = entry.WriteIdempotent }; else { _skippedTags.Skip(entry.RawPath, "TagConfig did not map to a definition"); @@ -365,30 +544,76 @@ public sealed class ModbusDriver } } _skippedTags.Commit(); - WriteHealth(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); - // PR 23: kick off the probe loop once the transport is up. Initial state stays - // Unknown until the first probe tick succeeds — avoids broadcasting a premature - // Running transition before any register round-trip has happened. - if (_options.Probe.Enabled) + // Publish transport + table + Healthy as ONE generation-checked step. A superseded attempt + // (a shutdown/initialize ran while this one was abandoned mid-connect) publishes nothing and + // releases its own transport. + lock (_stateLock) { - _probeCts = new CancellationTokenSource(); - _ = Task.Run(() => ProbeLoopAsync(_probeCts.Token), _probeCts.Token); - } + if (generation != _generation) + throw new OperationCanceledException( + "Modbus initialize was superseded by a later lifecycle operation; its transport was released"); + _transport = transport; + Volatile.Write(ref _tagsByRawPath, table); + Volatile.Write(ref _refusalWarned, new ConcurrentDictionary(StringComparer.Ordinal)); + WriteHealth(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); - if (_options.AutoProhibitReprobeInterval is not null) - { - _reprobeCts = new CancellationTokenSource(); - _ = Task.Run(() => ReprobeLoopAsync(_reprobeCts.Token), _reprobeCts.Token); + // PR 23: kick off the probe loop once the transport is up. Initial state stays + // Unknown until the first probe tick succeeds — avoids broadcasting a premature + // Running transition before any register round-trip has happened. Started inside the + // generation-checked publish so a teardown that bypassed the lifecycle gate can never + // run between the publish and the loop start and strand a loop. + if (_options.Probe.Enabled) + { + var cts = new CancellationTokenSource(); + var probeTransport = transport; + _probeCts = cts; + _probeTask = Task.Run(() => ProbeLoopAsync(generation, probeTransport, cts.Token), cts.Token); + } + + if (_options.AutoProhibitReprobeInterval is not null) + { + var cts = new CancellationTokenSource(); + _reprobeCts = cts; + _reprobeTask = Task.Run(() => ReprobeLoopAsync(cts.Token), cts.Token); + } } + transport = null; // published — owned by the driver now } catch (Exception ex) { - WriteHealth(new DriverHealth(DriverState.Faulted, null, ex.Message)); + // A failed attempt releases the transport it built — it is never left half-published. + if (transport is not null) + { + try { await transport.DisposeAsync().ConfigureAwait(false); } catch { } + } + WriteLifecycleHealth(generation, new DriverHealth(DriverState.Faulted, null, ex.Message)); throw; } } + /// Cancels, bounded-joins and disposes the probe and re-probe loops. Idempotent. + private async Task StopBackgroundLoopsAsync() + { + var probeCts = _probeCts; + var reprobeCts = _reprobeCts; + var tasks = new[] { _probeTask, _reprobeTask }.Where(t => t is not null).Cast().ToArray(); + _probeCts = null; + _reprobeCts = null; + _probeTask = null; + _reprobeTask = null; + + try { probeCts?.Cancel(); } catch { } + try { reprobeCts?.Cancel(); } catch { } + if (tasks.Length > 0) + { + try { await Task.WhenAll(tasks).WaitAsync(BackgroundLoopJoinTimeout).ConfigureAwait(false); } + catch { /* a loop that ignores its token is abandoned, never awaited forever */ } + } + probeCts?.Dispose(); + reprobeCts?.Dispose(); + } + /// public async Task ReinitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { @@ -435,9 +660,17 @@ public sealed class ModbusDriver /// . private async Task ShutdownCoreAsync() { - var lastRead = ReadHealth().LastSuccessfulRead; + int generation; + DateTime? lastRead; + lock (_stateLock) + { + // A shutdown is a lifecycle operation too: it supersedes any in-flight (abandoned) initialize, + // and its own final write below is dropped if a NEWER initialize started meanwhile. + generation = ++_generation; + lastRead = _health.LastSuccessfulRead; + } await TeardownAsync().ConfigureAwait(false); - WriteHealth(new DriverHealth(DriverState.Unknown, lastRead, null)); + WriteLifecycleHealth(generation, new DriverHealth(DriverState.Unknown, lastRead, null)); } /// @@ -489,57 +722,83 @@ public sealed class ModbusDriver public async Task> ReadAsync( IReadOnlyList fullReferences, CancellationToken cancellationToken) { - var transport = RequireTransport(); + var (generation, transport, table) = CaptureLifecycle(); var now = DateTime.UtcNow; var results = new DataValueSnapshot[fullReferences.Count]; + var batch = new BatchOutcome(); var coalesced = _options.MaxReadGap > 0 - ? await ReadCoalescedAsync(transport, fullReferences, results, now, cancellationToken).ConfigureAwait(false) + ? await ReadCoalescedAsync(transport, table, fullReferences, results, now, batch, cancellationToken).ConfigureAwait(false) : new HashSet(); for (var i = 0; i < fullReferences.Count; i++) { if (coalesced.Contains(i)) continue; - if (!_resolver.TryResolve(fullReferences[i], out var tag)) + if (!table.TryGetValue(fullReferences[i], out var tag)) { results[i] = new DataValueSnapshot(null, StatusBadNodeIdUnknown, null, now); continue; } + var unit = ResolveUnitId(tag); + if (batch.IsFailedFast(unit)) + { + // Fail fast: this unit (or the whole endpoint) already failed at the transport this batch; + // paying the same timeout once per remaining tag would stall the poll N-fold. + results[i] = new DataValueSnapshot(null, StatusBadCommunicationError, null, now); + continue; + } try { var value = await ReadOneAsync(transport, tag, cancellationToken).ConfigureAwait(false); results[i] = new DataValueSnapshot(value, 0u, now, now); - WriteHealth(new DriverHealth(DriverState.Healthy, now, null)); + batch.Answered = true; // Invalidate the WriteOnChangeOnly cache when the read returns a different value // — typically an HMI-side or PLC-internal change. Without this, a setpoint // tweaked at the panel could be silently re-suppressed when our client tried // to restore it. - if (_options.WriteOnChangeOnly) - { - lock (_lastWrittenLock) - { - if (_lastWrittenByRef.TryGetValue(fullReferences[i], out var prev) && !Equals(prev, value)) - _lastWrittenByRef.Remove(fullReferences[i]); - } - } + InvalidateWriteCacheIfDiverged(fullReferences[i], value); } - catch (ModbusException mex) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - results[i] = new DataValueSnapshot(null, MapModbusExceptionToStatus(mex.ExceptionCode), null, now); - WriteHealth(new DriverHealth(DriverState.Degraded, ReadHealth().LastSuccessfulRead, mex.Message)); + throw; } catch (Exception ex) { - // Non-Modbus-layer failure: socket dropped, timeout, malformed response. Surface - // as communication error so callers can distinguish it from tag-level faults. - results[i] = new DataValueSnapshot(null, StatusBadCommunicationError, null, now); - WriteHealth(new DriverHealth(DriverState.Degraded, ReadHealth().LastSuccessfulRead, ex.Message)); + results[i] = new DataValueSnapshot(null, ClassifyFailure(ex, unit, fullReferences[i], batch, isWrite: false), null, now); } } + + // ONE health write per batch, worst-wins over the batch, generation-checked. It used to be + // last-writer-wins per tag, so health flapped with tag order. + ReportOutcome(generation, HealthSide.Read, batch, now); return results; } + /// + /// Captures the generation, transport and tag table a read/write batch runs against, atomically + /// with respect to a lifecycle publish. + /// + private (int Generation, IModbusTransport Transport, Dictionary Table) CaptureLifecycle() + { + lock (_stateLock) + { + var transport = _transport ?? throw new InvalidOperationException("ModbusDriver not initialized"); + return (_generation, transport, _tagsByRawPath); + } + } + + /// Runs a codec decode, normalising its failure to so the + /// classifier can tell undecodable device data from a transport fault. + private static object Decode(Func decode, ModbusTagDefinition tag) + { + try { return decode(); } + catch (Exception ex) when (ex is InvalidDataException or OverflowException or ArgumentException or InvalidCastException) + { + throw new ModbusDecodeException($"value of {tag.Name} ({tag.DataType}) could not be decoded: {ex.Message}", ex); + } + } + private async Task ReadOneAsync(IModbusTransport transport, ModbusTagDefinition tag, CancellationToken ct) { var arrayCount = tag.ArrayCount ?? 1; @@ -577,9 +836,9 @@ public sealed class ModbusDriver : await ReadRegisterBlockChunkedAsync(transport, unitId, fc, tag.Address, totalRegs, cap, ct).ConfigureAwait(false); if (!tag.ArrayCount.HasValue) - return DecodeRegister(data, tag); + return Decode(() => DecodeRegister(data, tag), tag); - return DecodeRegisterArray(data, tag, elementRegs, arrayCount); + return Decode(() => DecodeRegisterArray(data, tag, elementRegs, arrayCount), tag); } default: throw new InvalidOperationException($"Unknown region {tag.Region}"); @@ -913,9 +1172,11 @@ public sealed class ModbusDriver /// private async Task> ReadCoalescedAsync( IModbusTransport transport, + Dictionary table, IReadOnlyList fullReferences, DataValueSnapshot[] results, DateTime timestamp, + BatchOutcome batch, CancellationToken ct) { // Eligible: known tag, register region (HoldingRegisters or InputRegisters), scalar @@ -923,7 +1184,7 @@ public sealed class ModbusDriver var eligible = new List<(int Index, string Ref, ModbusTagDefinition Tag)>(); for (var i = 0; i < fullReferences.Count; i++) { - if (!_resolver.TryResolve(fullReferences[i], out var tag)) continue; + if (!table.TryGetValue(fullReferences[i], out var tag)) continue; if (tag.CoalesceProhibited) continue; if (tag.ArrayCount.HasValue) continue; if (tag.Region is not (ModbusRegion.HoldingRegisters or ModbusRegion.InputRegisters)) continue; @@ -983,38 +1244,66 @@ public sealed class ModbusDriver } var qty = (ushort)(block.End - block.Start + 1); - try + if (batch.IsFailedFast(group.Key.Unit)) { - var data = await ReadRegisterBlockAsync(transport, group.Key.Unit, fc, block.Start, qty, ct).ConfigureAwait(false); - foreach (var (idx, tag) in block.Members) - { - var sliceOffsetBytes = (tag.Address - block.Start) * 2; - var sliceLenBytes = RegisterCount(tag) * 2; - var value = DecodeRegister(data.AsSpan(sliceOffsetBytes, sliceLenBytes), tag); - results[idx] = new DataValueSnapshot(value, 0u, timestamp, timestamp); - handled.Add(idx); - InvalidateWriteCacheIfDiverged(fullReferences[idx], value); - } - WriteHealth(new DriverHealth(DriverState.Healthy, timestamp, null)); - } - catch (ModbusException mex) - { - RecordAutoProhibition(group.Key.Unit, group.Key.Region, block.Start, block.End); - WriteHealth(new DriverHealth(DriverState.Degraded, ReadHealth().LastSuccessfulRead, mex.Message)); - } - catch (Exception ex) - { - // Communication failures (timeout, socket drop) aren't a structural reason - // to prohibit the range — the same coalesced read might succeed once the - // transport recovers. Mark members Bad for this scan but don't auto-prohibit - // and don't deflect to per-tag fallback (which would just hit the same dead - // socket). foreach (var (idx, _) in block.Members) { results[idx] = new DataValueSnapshot(null, StatusBadCommunicationError, null, timestamp); handled.Add(idx); } - WriteHealth(new DriverHealth(DriverState.Degraded, ReadHealth().LastSuccessfulRead, ex.Message)); + continue; + } + byte[] data; + try + { + data = await ReadRegisterBlockAsync(transport, group.Key.Unit, fc, block.Start, qty, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (ModbusException mex) when (IsRefusalCode(mex.ExceptionCode)) + { + // The device refused the merged range (a protected hole inside it): record the range + // as auto-prohibited and leave the members UNhandled so the per-tag fallback reads each + // surviving address individually. The device answered, so health is not degraded. + batch.Answered = true; + RecordAutoProhibition(group.Key.Unit, group.Key.Region, block.Start, block.End); + continue; + } + catch (Exception ex) + { + // Communication failures (timeout, socket drop) and non-refusal exception codes aren't + // a structural reason to prohibit the range — the same coalesced read might succeed + // once the transport recovers. Mark members Bad for this scan but don't auto-prohibit + // and don't deflect to per-tag fallback (which would just hit the same dead socket). + var status = ClassifyFailure(ex, group.Key.Unit, fullReferences[block.Members[0].Index], batch, isWrite: false); + foreach (var (idx, _) in block.Members) + { + results[idx] = new DataValueSnapshot(null, status, null, timestamp); + handled.Add(idx); + } + continue; + } + + batch.Answered = true; + foreach (var (idx, tag) in block.Members) + { + var sliceOffsetBytes = (tag.Address - block.Start) * 2; + var sliceLenBytes = RegisterCount(tag) * 2; + handled.Add(idx); + try + { + var value = Decode(() => DecodeRegister(data.AsSpan(sliceOffsetBytes, sliceLenBytes), tag), tag); + results[idx] = new DataValueSnapshot(value, 0u, timestamp, timestamp); + InvalidateWriteCacheIfDiverged(fullReferences[idx], value); + } + catch (ModbusDecodeException dex) + { + // One undecodable member never fails its neighbours. + results[idx] = new DataValueSnapshot(null, + ClassifyFailure(dex, group.Key.Unit, fullReferences[idx], batch, isWrite: false), null, timestamp); + } } } } @@ -1124,12 +1413,13 @@ public sealed class ModbusDriver public async Task> WriteAsync( IReadOnlyList writes, CancellationToken cancellationToken) { - var transport = RequireTransport(); + var (generation, transport, table) = CaptureLifecycle(); var results = new WriteResult[writes.Count]; + var batch = new BatchOutcome(); for (var i = 0; i < writes.Count; i++) { var w = writes[i]; - if (!_resolver.TryResolve(w.FullReference, out var tag)) + if (!table.TryGetValue(w.FullReference, out var tag)) { results[i] = new WriteResult(StatusBadNodeIdUnknown); continue; @@ -1144,30 +1434,67 @@ public sealed class ModbusDriver results[i] = new WriteResult(0u); continue; } + var unit = ResolveUnitId(tag); + if (batch.IsFailedFast(unit)) + { + results[i] = new WriteResult(StatusBadCommunicationError); + continue; + } try { await WriteOneAsync(transport, tag, w.Value, cancellationToken).ConfigureAwait(false); results[i] = new WriteResult(0u); + batch.Answered = true; if (_options.WriteOnChangeOnly) lock (_lastWrittenLock) _lastWrittenByRef[w.FullReference] = w.Value; } - catch (ModbusException mex) + catch (Exception ex) { - results[i] = new WriteResult(MapModbusExceptionToStatus(mex.ExceptionCode)); - } - catch (InvalidDataException) - { - results[i] = new WriteResult(StatusBadCommunicationError); - } - catch (Exception) - { - results[i] = new WriteResult(StatusBadInternalError); + results[i] = new WriteResult(MapWriteFailure(ex, unit, w.FullReference, batch, cancellationToken)); } } + + // Write-side health: a comms failure degrades it (LastSuccessfulRead preserved); a good write or a + // device refusal proves the device answered and clears a write-side Degraded. Generation-checked. + ReportOutcome(generation, HealthSide.Write, batch, DateTime.UtcNow); return results; } + /// + /// Write-status mapping (review 07 H7). Failures raised BEFORE the wire — a value that does not fit + /// (BadOutOfRange), cannot convert (BadTypeMismatch) or is not supported + /// (BadNotSupported) — never touch health. Device answers go through + /// (refusal / device fault / comms fault). Transport failures — + /// desync, timeout, socket, EOF — are BadCommunicationError, no longer + /// BadInternalError; only a genuinely unexpected exception still is. + /// + private uint MapWriteFailure(Exception ex, byte unit, string tagRef, BatchOutcome batch, CancellationToken ct) + { + switch (ex) + { + case OperationCanceledException when ct.IsCancellationRequested: + // The caller's write deadline fired (e.g. waiting on the single-flight gate). The device + // said nothing, so health is untouched. + return StatusBadTimeout; + case ModbusWriteRefusedException refused: + return refused.StatusCode; + case OverflowException or ArgumentOutOfRangeException: + return StatusBadOutOfRange; + case FormatException or InvalidCastException or ArgumentException: + return StatusBadTypeMismatch; + case NotSupportedException: + return StatusBadNotSupported; + case ModbusException or ModbusDecodeException or IOException or InvalidDataException + or System.Net.Sockets.SocketException or ObjectDisposedException or TimeoutException + or OperationCanceledException: + return ClassifyFailure(ex, unit, tagRef, batch, isWrite: true); + default: + _logger.LogWarning(ex, "Modbus write to {RawPath} failed unexpectedly. Driver={DriverInstanceId}", tagRef, _driverInstanceId); + return StatusBadInternalError; + } + } + private bool IsRedundantWrite(string tagRef, object? value) { lock (_lastWrittenLock) @@ -1256,7 +1583,7 @@ public sealed class ModbusDriver var qty = (ushort)(bytes.Length / 2); var writeCap = _options.MaxRegistersPerWrite == 0 ? (ushort)123 : _options.MaxRegistersPerWrite; if (qty > writeCap) - throw new InvalidOperationException( + throw new ModbusWriteRefusedException(StatusBadOutOfRange, $"Write of {qty} registers to {tag.Name} exceeds MaxRegistersPerWrite={writeCap}. " + $"Split the tag (e.g. shorter StringLength or smaller ArrayCount) — partial FC16 chunks would lose atomicity."); var pdu = new byte[6 + 1 + bytes.Length]; @@ -1270,7 +1597,7 @@ public sealed class ModbusDriver return; } default: - throw new InvalidOperationException($"Writes not supported for region {tag.Region}"); + throw new ModbusWriteRefusedException(StatusBadNotWritable, $"Writes not supported for region {tag.Region}"); } } @@ -1281,9 +1608,12 @@ public sealed class ModbusDriver private static byte[] EncodeRegisterArray(object? value, ModbusTagDefinition tag) { var count = tag.ArrayCount!.Value; - if (value is not System.Collections.IList list || list.Count != count) - throw new InvalidOperationException( + if (value is not System.Collections.IList list) + throw new ModbusWriteRefusedException(StatusBadTypeMismatch, $"Array write to {tag.Name} expects an IList of length {count}; got {value?.GetType().Name ?? "null"}"); + if (list.Count != count) + throw new ModbusWriteRefusedException(StatusBadOutOfRange, + $"Array write to {tag.Name} expects {count} elements; got {list.Count}"); var elementBytes = ElementByteCount(tag); var result = new byte[count * elementBytes]; @@ -1308,7 +1638,10 @@ public sealed class ModbusDriver for (var i = 0; i < expectedCount; i++) arr[i] = Convert.ToBoolean(list[i]); return arr; } - throw new InvalidOperationException( + if (value is System.Collections.IList wrong) + throw new ModbusWriteRefusedException(StatusBadOutOfRange, + $"Coil-array write to {tagName} expects {expectedCount} elements; got {wrong.Count}"); + throw new ModbusWriteRefusedException(StatusBadTypeMismatch, $"Coil-array write to {tagName} expects a bool[] (or convertible IList) of length {expectedCount}; got {value?.GetType().Name ?? "null"}"); } @@ -1392,12 +1725,12 @@ public sealed class ModbusDriver /// public string HostName => $"{_options.Host}:{_options.Port}"; - private async Task ProbeLoopAsync(CancellationToken ct) + private async Task ProbeLoopAsync(int generation, IModbusTransport transport, CancellationToken ct) { - var transport = _transport; // captured reference; disposal tears the loop down via ct + // The transport is captured at start; disposal tears the loop down via ct. while (!ct.IsCancellationRequested) { - var success = false; + var answered = false; try { using var probeCts = CancellationTokenSource.CreateLinkedTokenSource(ct); @@ -1405,19 +1738,28 @@ public sealed class ModbusDriver var pdu = new byte[] { 0x03, (byte)(_options.Probe.ProbeAddress >> 8), (byte)(_options.Probe.ProbeAddress & 0xFF), 0x00, 0x01 }; - _ = await transport!.SendAsync(_options.UnitId, pdu, probeCts.Token).ConfigureAwait(false); - success = true; + _ = await transport.SendAsync(_options.UnitId, pdu, probeCts.Token).ConfigureAwait(false); + answered = true; } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } + catch (ModbusException mex) when (mex.ExceptionCode is not (0x0A or 0x0B)) + { + // An exception PDU from the device itself (e.g. a locked probe register) proves the host + // is up — it answered. Only a gateway reporting its target unreachable (0A/0B) is Stopped. + answered = true; + } catch { - // transport / timeout / exception PDU — treated as Stopped below + // transport / timeout / gateway-target failure — treated as Stopped below } - TransitionTo(success ? HostState.Running : HostState.Stopped); + // A probe that finished after teardown cancelled the loop must not raise a stale transition. + if (ct.IsCancellationRequested) return; + if (answered) NoteDeviceAnswered(generation); + TransitionTo(answered ? HostState.Running : HostState.Stopped); try { await Task.Delay(_options.Probe.Interval, ct).ConfigureAwait(false); } catch (OperationCanceledException) { return; } @@ -1743,8 +2085,6 @@ public sealed class ModbusDriver return result; } - private IModbusTransport RequireTransport() => - _transport ?? throw new InvalidOperationException("ModbusDriver not initialized"); private const uint StatusBadInternalError = 0x80020000u; private const uint StatusBadNodeIdUnknown = 0x80340000u; @@ -1753,6 +2093,9 @@ public sealed class ModbusDriver private const uint StatusBadNotSupported = 0x803D0000u; private const uint StatusBadDeviceFailure = 0x808B0000u; private const uint StatusBadCommunicationError = 0x80050000u; + private const uint StatusBadDecodingError = 0x80070000u; + private const uint StatusBadTimeout = 0x800A0000u; + private const uint StatusBadTypeMismatch = 0x80740000u; /// /// Map a server-returned Modbus exception code to the most informative OPC UA @@ -1789,6 +2132,10 @@ public sealed class ModbusDriver /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { + // Dispose supersedes any in-flight (abandoned) initialize up front, even when it then has to + // bypass the gate: that initialize will find its generation moved on and release its own + // transport instead of publishing it after this teardown. + lock (_stateLock) _generation++; if (await _lifecycleGate.WaitAsync(LifecycleGateTimeout).ConfigureAwait(false)) { try @@ -1816,25 +2163,27 @@ public sealed class ModbusDriver /// private async Task TeardownAsync() { - try { _probeCts?.Cancel(); } catch { } - _probeCts?.Dispose(); - _probeCts = null; + await StopBackgroundLoopsAsync().ConfigureAwait(false); - try { _reprobeCts?.Cancel(); } catch { } - _reprobeCts?.Dispose(); - _reprobeCts = null; + // Drain the poll loops BEFORE dropping the tables and caches they read, so no in-flight cycle can + // re-seed a cache after it was cleared. + await _poll.DisposeAsync().ConfigureAwait(false); + // After the engine's loops are drained, no onChange can re-populate the per-cycle buffers. + _cycleBatchItems.Clear(); - _tagsByRawPath.Clear(); + IModbusTransport? transport; + lock (_stateLock) + { + transport = _transport; + _transport = null; + // Swap, never .Clear() in place: a reader that captured the old table keeps a consistent view. + Volatile.Write(ref _tagsByRawPath, new Dictionary(StringComparer.Ordinal)); + } _resolver.Clear(); // no-op in v3 (the resolver reads the live table by closure); kept for symmetry _lastPublishedByRef.Clear(); lock (_lastWrittenLock) _lastWrittenByRef.Clear(); lock (_autoProhibitedLock) _autoProhibited.Clear(); - await _poll.DisposeAsync().ConfigureAwait(false); - // After the engine's loops are drained, no onChange can re-populate the per-cycle buffers. - _cycleBatchItems.Clear(); - - if (_transport is not null) await _transport.DisposeAsync().ConfigureAwait(false); - _transport = null; + if (transport is not null) await transport.DisposeAsync().ConfigureAwait(false); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs index 69ccf915..7dcc5dff 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuOverTcpTransport.cs @@ -46,7 +46,7 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport private readonly Stream? _testStream; private readonly TimeSpan _timeout; private readonly SemaphoreSlim _gate = new(1, 1); - private bool _disposed; + private int _disposed; /// Initializes a new instance of the class. /// The host address or hostname of the Modbus gateway. @@ -90,7 +90,7 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport /// public async Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) { - if (_disposed) throw new ObjectDisposedException(nameof(ModbusRtuOverTcpTransport)); + if (Volatile.Read(ref _disposed) != 0) throw new ObjectDisposedException(nameof(ModbusRtuOverTcpTransport)); if (_life is null && _testStream is null) throw new InvalidOperationException("Transport not connected"); // Single-flight: RTU carries no transaction id, so at most one transaction may be on the bus @@ -187,9 +187,8 @@ public sealed class ModbusRtuOverTcpTransport : IModbusTransport /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { - if (_disposed) return; - _disposed = true; + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; if (_life is not null) await _life.DisposeAsync().ConfigureAwait(false); - _gate.Dispose(); + // _gate is deliberately NOT disposed: a SendAsync still in flight at teardown releases it in its finally. } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs index 7a032cdd..34fcbdd3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs @@ -76,6 +76,7 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable { // A previous socket (failed, torn down or idle-reaped) is released FIRST: overwriting _client // used to strand the old TcpClient for the finalizer — up to nine per backoff loop. + ThrowIfDisposed(); await TearDownAsync().ConfigureAwait(false); // DNS resolution runs under the same per-op deadline as the TCP connect: an unresolvable or @@ -98,6 +99,7 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable client = new TcpClient(target.AddressFamily); EnableKeepAlive(client, _keepAlive); await client.ConnectAsync(target, _port, cts.Token).ConfigureAwait(false); + ThrowIfDisposed(); // disposed while dialling — the finally releases the fresh client _stream = client.GetStream(); _client = client; client = null; // ownership transferred @@ -258,11 +260,16 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { - try - { - if (_stream is not null) await _stream.DisposeAsync().ConfigureAwait(false); - } - catch { /* best-effort */ } - _client?.Dispose(); + // Latch first: a SendAsync still in flight (the driver tearing down under it) must not reconnect + // a fresh socket after this point — ConnectAsync checks the latch both before and after dialling. + Volatile.Write(ref _disposed, 1); + await TearDownAsync().ConfigureAwait(false); + } + + private int _disposed; + + private void ThrowIfDisposed() + { + if (Volatile.Read(ref _disposed) != 0) throw new ObjectDisposedException(nameof(ModbusSocketLifecycle)); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs index 26534610..f21ce678 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusTcpTransport.cs @@ -36,7 +36,7 @@ public sealed class ModbusTcpTransport : IModbusTransport private readonly TimeSpan _timeout; private readonly SemaphoreSlim _gate = new(1, 1); private ushort _nextTx; - private bool _disposed; + private int _disposed; /// Initializes a new instance of the class. /// The host address or hostname of the Modbus server. @@ -65,7 +65,7 @@ public sealed class ModbusTcpTransport : IModbusTransport /// public async Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) { - if (_disposed) throw new ObjectDisposedException(nameof(ModbusTcpTransport)); + if (Volatile.Read(ref _disposed) != 0) throw new ObjectDisposedException(nameof(ModbusTcpTransport)); await _gate.WaitAsync(ct).ConfigureAwait(false); try @@ -228,9 +228,8 @@ public sealed class ModbusTcpTransport : IModbusTransport /// A task that represents the asynchronous operation. public async ValueTask DisposeAsync() { - if (_disposed) return; - _disposed = true; + if (Interlocked.Exchange(ref _disposed, 1) != 0) return; await _life.DisposeAsync().ConfigureAwait(false); - _gate.Dispose(); + // _gate is deliberately NOT disposed: a SendAsync still in flight at teardown releases it in its finally. } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusWriteRefusedException.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusWriteRefusedException.cs new file mode 100644 index 00000000..866990ec --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusWriteRefusedException.cs @@ -0,0 +1,21 @@ +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus; + +/// +/// The driver refused a write BEFORE anything reached the wire (wrong array length, a payload over +/// MaxRegistersPerWrite, an unsupported shape). Carries the OPC UA status the write returns; +/// it never touches driver health — nothing was sent. +/// +public sealed class ModbusWriteRefusedException : Exception +{ + /// Initializes a new instance of the class. + /// The OPC UA status the write returns. + /// A human-readable description. + public ModbusWriteRefusedException(uint statusCode, string message) + : base(message) + { + StatusCode = statusCode; + } + + /// Gets the OPC UA status the write returns. + public uint StatusCode { get; } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs new file mode 100644 index 00000000..50282ec3 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs @@ -0,0 +1,341 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// The driver health contract (review 07 H4 + H7, and the lifecycle/fail-fast rules every driver lane +/// must meet): a device REFUSAL is per-tag Bad with no Degraded and proves the device answered; a comms +/// fault degrades and preserves LastSuccessfulRead; health is one write per batch; a comms timeout on +/// one unit fails the rest of that unit's batch fast; writes map transport failures to +/// BadCommunicationError and value problems to BadOutOfRange / BadTypeMismatch. +/// +[Trait("Category", "Unit")] +public sealed class ModbusHealthContractTests +{ + private const uint Good = 0u; + private const uint BadOutOfRange = 0x803C0000u; + private const uint BadDeviceFailure = 0x808B0000u; + private const uint BadCommunicationError = 0x80050000u; + private const uint BadTypeMismatch = 0x80740000u; + private const uint BadNodeIdUnknown = 0x80340000u; + + private static ModbusDriverOptions Opts(params ModbusTagDefinition[] tags) => new() + { + Host = "f", RawTags = ModbusRawTags.Entries(tags), Probe = new ModbusProbeOptions { Enabled = false }, + }; + + private static ModbusTagDefinition Hr(string name, ushort addr, byte? unit = null, ModbusDataType type = ModbusDataType.Int16) => + new(name, ModbusRegion.HoldingRegisters, addr, type, UnitId: unit); + + private static Task Refuse(byte code, byte[] pdu) => + Task.FromException(new ModbusException(pdu[0], code, $"exception {code}")); + + private static ushort Addr(byte[] pdu) => (ushort)((pdu[1] << 8) | pdu[2]); + + private static async Task<(ModbusDriver Driver, ScriptedTransport Transport)> StartAsync( + ModbusDriverOptions opts, ILogger? logger = null) + { + var t = new ScriptedTransport(); + var d = new ModbusDriver(opts, "m1", _ => t, logger); + await d.InitializeAsync("{}", TestContext.Current.CancellationToken); + return (d, t); + } + + // ---------------- H4: refusals ---------------- + + [Fact] + public async Task All_tags_refused_keeps_Healthy_and_advances_LastSuccessfulRead() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10), Hr("B", 11))); + await using var _ = drv; + t.Handler = (_, pdu, _) => Refuse(0x02, pdu); + var before = drv.GetHealth().LastSuccessfulRead; + + var r = await drv.ReadAsync(["A", "B"], TestContext.Current.CancellationToken); + + r.ShouldAllBe(s => s.StatusCode == BadOutOfRange); + var h = drv.GetHealth(); + h.State.ShouldBe(DriverState.Healthy, "a refusal means the device answered — no Degraded, no watchdog fodder"); + h.LastSuccessfulRead.ShouldNotBeNull(); + h.LastSuccessfulRead!.Value.ShouldBeGreaterThanOrEqualTo(before!.Value); + } + + [Fact] + public async Task A_refusal_after_a_comms_fault_restores_Healthy() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10))); + await using var _ = drv; + t.Handler = (_, _, _) => Task.FromException(new IOException("socket dropped")); + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + var degraded = drv.GetHealth(); + degraded.State.ShouldBe(DriverState.Degraded); + + t.Handler = (_, pdu, _) => Refuse(0x02, pdu); + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + var h = drv.GetHealth(); + h.State.ShouldBe(DriverState.Healthy); + h.LastSuccessfulRead!.Value.ShouldBeGreaterThanOrEqualTo(degraded.LastSuccessfulRead!.Value); + } + + [Theory] + [InlineData(0x04)] + [InlineData(0x0B)] + public async Task Non_refusal_exception_codes_still_degrade(byte code) + { + // Positive control for the refusal tests: a server-failure / gateway code is NOT a refusal. + var (drv, t) = await StartAsync(Opts(Hr("A", 10))); + await using var _ = drv; + t.Handler = (_, pdu, _) => Refuse(code, pdu); + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Mixed_refused_and_good_batch_is_Healthy_regardless_of_tag_order(bool refusedFirst) + { + var (drv, t) = await StartAsync(Opts(Hr("Refused", 10), Hr("Fine", 50))); + await using var _ = drv; + t.Handler = (_, pdu, _) => Addr(pdu) == 10 ? Refuse(0x02, pdu) : null; + string[] refs = refusedFirst ? ["Refused", "Fine"] : ["Fine", "Refused"]; + var r = await drv.ReadAsync(refs, TestContext.Current.CancellationToken); + r.Count(s => s.StatusCode == Good).ShouldBe(1); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + [Fact] + public async Task A_refused_tag_warns_once_not_once_per_poll() + { + var logger = new ListLogger(); + var (drv, t) = await StartAsync(Opts(Hr("A", 10)), logger); + await using var _ = drv; + t.Handler = (_, pdu, _) => Refuse(0x02, pdu); + for (var i = 0; i < 3; i++) await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + logger.Warnings.Count(m => m.Contains("refused") && m.Contains("A")).ShouldBe(1); + } + + // ---------------- fail fast per unit / endpoint ---------------- + + [Fact] + public async Task A_comms_timeout_on_one_unit_fails_the_rest_of_that_units_batch_fast() + { + var (drv, t) = await StartAsync(Opts(Hr("U2a", 10, 2), Hr("U2b", 11, 2), Hr("U2c", 12, 2), Hr("U1", 13, 1))); + await using var _ = drv; + t.Bank.HoldingRegisters[13] = 42; + t.Handler = (unit, _, _) => unit == 2 + ? Task.FromException(new ModbusTransportDesyncException( + ModbusTransportDesyncException.DesyncReason.Timeout, "timeout")) + : null; + + var r = await drv.ReadAsync(["U2a", "U2b", "U2c", "U1"], TestContext.Current.CancellationToken); + + t.SendsTo(2).ShouldBe(1, "one timeout per unit per batch, not one per tag"); + r.Take(3).ShouldAllBe(s => s.StatusCode == BadCommunicationError); + // Positive control: the healthy unit in the same batch is still read. + r[3].StatusCode.ShouldBe(Good); + r[3].Value.ShouldBe((short)42); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + [Fact] + public async Task An_unreachable_endpoint_fails_the_whole_batch_fast() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10, 1), Hr("B", 11, 2), Hr("C", 12, 3))); + await using var _ = drv; + t.Handler = (_, _, _) => Task.FromException(new ModbusTransportUnavailableException("throttled")); + var r = await drv.ReadAsync(["A", "B", "C"], TestContext.Current.CancellationToken); + t.TotalSends.ShouldBe(1); + r.ShouldAllBe(s => s.StatusCode == BadCommunicationError); + } + + // ---------------- H7: write status mapping + write-side health ---------------- + + [Fact] + public async Task Write_transport_failure_is_BadCommunicationError_and_degrades_the_write_side() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10))); + await using var _ = drv; + t.Handler = (_, pdu, _) => pdu[0] == 0x06 ? Task.FromException(new IOException("dropped")) : null; + + var w = await drv.WriteAsync([new WriteRequest("A", (short)1)], TestContext.Current.CancellationToken); + w[0].StatusCode.ShouldBe(BadCommunicationError); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + // A good READ does not clear a write-side fault... + (await drv.ReadAsync(["A"], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(Good); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + // ...a good WRITE does. + t.Handler = null; + (await drv.WriteAsync([new WriteRequest("A", (short)2)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(Good); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + [Fact] + public async Task A_write_refusal_counts_as_answered_and_clears_a_write_side_fault() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10))); + await using var _ = drv; + t.Handler = (_, pdu, _) => pdu[0] == 0x06 ? Task.FromException(new IOException("dropped")) : null; + await drv.WriteAsync([new WriteRequest("A", (short)1)], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + + t.Handler = (_, pdu, _) => pdu[0] == 0x06 ? Refuse(0x03, pdu) : null; + var w = await drv.WriteAsync([new WriteRequest("A", (short)1)], TestContext.Current.CancellationToken); + w[0].StatusCode.ShouldBe(BadOutOfRange); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + [Fact] + public async Task Write_value_problems_map_to_OutOfRange_and_TypeMismatch_without_touching_health() + { + var arr = new ModbusTagDefinition("Arr", ModbusRegion.HoldingRegisters, 20, ModbusDataType.Int16, ArrayCount: 3); + var (drv, t) = await StartAsync(Opts(Hr("Bcd", 10, type: ModbusDataType.Bcd16), Hr("I16", 11), arr)); + await using var _ = drv; + + var w = await drv.WriteAsync([ + new WriteRequest("Bcd", 12345), // > 4 BCD digits + new WriteRequest("I16", 70000), // > Int16 + new WriteRequest("I16", "not a number"), + new WriteRequest("Arr", new short[] { 1, 2 }), // length mismatch + new WriteRequest("Arr", new int[] { 1, 2, 3 }), // CLR list of another element type converts (ScadaBridge shape) + ], TestContext.Current.CancellationToken); + + w[0].StatusCode.ShouldBe(BadOutOfRange); + w[1].StatusCode.ShouldBe(BadOutOfRange); + w[2].StatusCode.ShouldBe(BadTypeMismatch); + w[3].StatusCode.ShouldBe(BadOutOfRange); + w[4].StatusCode.ShouldBe(Good); + t.Bank.HoldingRegisters[22].ShouldBe((ushort)3); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + // ---------------- lifecycle generation ---------------- + + [Fact] + public async Task A_superseded_initialize_never_publishes_over_the_shutdown_that_overtook_it() + { + var t = new ScriptedTransport { ConnectGate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously) }; + var drv = new ModbusDriver(Opts(Hr("A", 10)), "m1", _ => t) { LifecycleGateTimeout = TimeSpan.FromSeconds(1) }; + var init = drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await t.ConnectEntered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + // The init is wedged in its connect; shutdown waits out the gate timeout and tears down anyway. + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Unknown); + + t.ConnectGate.SetResult(); // the abandoned connect finally completes + await Should.ThrowAsync(() => init); + + drv.GetHealth().State.ShouldBe(DriverState.Unknown, "the superseded init must not publish Healthy"); + t.DisposeCount.ShouldBe(1, "the superseded init releases the transport it built"); + await Should.ThrowAsync(() => drv.ReadAsync(["A"], TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task A_read_that_straddles_a_reinitialize_does_not_write_health_about_the_new_transport() + { + var first = new ScriptedTransport(); + var second = new ScriptedTransport(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + first.Handler = (_, _, _) => { entered.TrySetResult(); return release.Task; }; + var calls = 0; + var drv = new ModbusDriver(Opts(Hr("A", 10)), "m1", _ => Interlocked.Increment(ref calls) == 1 ? first : second); + await using var _ = drv; + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var read = drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); // re-init onto `second` + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + + release.SetException(new IOException("old socket died")); // the straddling read now fails + (await read)[0].StatusCode.ShouldBe(BadCommunicationError); + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a read on the superseded transport must not degrade the new one"); + + // Positive control: a failure on the CURRENT transport still degrades. + second.Handler = (_, _, _) => Task.FromException(new IOException("new socket died")); + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + drv.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + [Fact] + public async Task Reinitialize_with_a_new_tag_set_drops_tags_the_new_config_no_longer_has() + { + var t = new ScriptedTransport(); + var drv = new ModbusDriver(Opts(Hr("Old", 10)), "m1", _ => t); + await using var _ = drv; + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + (await drv.ReadAsync(["Old"], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(Good); + + var config = JsonSerializer.Serialize(new + { + host = "f", + probe = new { enabled = false }, + rawTags = ModbusRawTags.Entries([Hr("New", 11)]), + }); + await drv.InitializeAsync(config, TestContext.Current.CancellationToken); + + var r = await drv.ReadAsync(["Old", "New"], TestContext.Current.CancellationToken); + r[0].StatusCode.ShouldBe(BadNodeIdUnknown, "a re-init publishes a fresh table, it does not merge into the old one"); + r[1].StatusCode.ShouldBe(Good); + } + + // ---------------- complete teardown ---------------- + + [Fact] + public async Task Shutdown_joins_the_probe_loop_so_no_host_transition_fires_after_it_returns() + { + var t = new ScriptedTransport(); + var probeBlocked = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var probes = 0; + t.Handler = async (_, pdu, ct) => + { + if (Interlocked.Increment(ref probes) == 1) return await t.Bank.SendAsync(1, pdu, ct); // first probe: Running + probeBlocked.TrySetResult(); + try { await Task.Delay(Timeout.Infinite, ct); } + catch (OperationCanceledException) { } + throw new ObjectDisposedException("transport"); // a teardown-time failure, NOT an OCE + }; + var opts = new ModbusDriverOptions + { + Host = "f", RawTags = ModbusRawTags.Entries([Hr("A", 10)]), + Probe = new ModbusProbeOptions { Enabled = true, Interval = TimeSpan.FromMilliseconds(50), Timeout = TimeSpan.FromSeconds(30) }, + }; + var drv = new ModbusDriver(opts, "m1", _ => t); + var running = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var afterShutdown = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var shutdownReturned = false; + drv.OnHostStatusChanged += (_, e) => + { + if (e.NewState == HostState.Running) running.TrySetResult(); + if (Volatile.Read(ref shutdownReturned)) afterShutdown.TrySetResult(); + }; + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + await running.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); // positive control + await probeBlocked.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + await drv.ShutdownAsync(TestContext.Current.CancellationToken); + Volatile.Write(ref shutdownReturned, true); + + await Should.ThrowAsync( + () => afterShutdown.Task.WaitAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); + } + + private sealed class ListLogger : ILogger + { + public System.Collections.Concurrent.ConcurrentQueue Messages { get; } = new(); + public IEnumerable Warnings => Messages.Where(m => m.StartsWith("Warning:", StringComparison.Ordinal)); + public IDisposable? BeginScope(TState state) where TState : notnull => null; + public bool IsEnabled(LogLevel logLevel) => true; + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + => Messages.Enqueue($"{logLevel}: {formatter(state, exception)}"); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusLoggerInjectionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusLoggerInjectionTests.cs index f4caef3f..867b20ee 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusLoggerInjectionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusLoggerInjectionTests.cs @@ -92,7 +92,7 @@ public sealed class ModbusLoggerInjectionTests // Scan 1 — coalesced read fails. Expect exactly one warning. await drv.ReadAsync(["T100", "T102", "T104"], CancellationToken.None); - var warnings = logger.Entries.Where(e => e.Level == LogLevel.Warning).ToList(); + var warnings = logger.Entries.Where(e => e.Level == LogLevel.Warning && e.Message.Contains("auto-prohibited")).ToList(); warnings.Count.ShouldBe(1); warnings[0].Message.ShouldContain("drv-logged"); warnings[0].Message.ShouldContain("Start=100"); @@ -102,7 +102,13 @@ public sealed class ModbusLoggerInjectionTests // the prohibition and skips the merge; even if it didn't, the de-dupe in // RecordAutoProhibition would suppress). await drv.ReadAsync(["T100", "T102", "T104"], CancellationToken.None); - logger.Entries.Count(e => e.Level == LogLevel.Warning).ShouldBe(1, "re-fire of same range stays silent"); + logger.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("auto-prohibited")) + .ShouldBe(1, "re-fire of same range stays silent"); + // The protected register's per-tag fallback is a device refusal: one Warning for that tag across + // both scans (once-per-tag), and none for its readable neighbours. + var refusals = logger.Entries.Where(e => e.Level == LogLevel.Warning && e.Message.Contains("refused")).ToList(); + refusals.Count.ShouldBe(1); + refusals[0].Message.ShouldContain("T102"); await drv.ShutdownAsync(CancellationToken.None); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs new file mode 100644 index 00000000..e865bc34 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs @@ -0,0 +1,51 @@ +using System.Collections.Concurrent; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// In-memory transport whose every send is answered by a test-supplied handler; falls back to the +/// register/coil bank fake () when the handler returns null. +/// Counts sends per unit and dispose calls; the connect can be blocked on a gate. +/// +internal sealed class ScriptedTransport : IModbusTransport +{ + private readonly ConcurrentDictionary _sendsByUnit = new(); + private int _disposeCount; + + /// The bank answering un-scripted sends. + public ModbusDriverTests.FakeTransport Bank { get; } = new(); + + /// Per-send handler; return null to fall through to . + public Func?>? Handler { get; set; } + + /// When set, waits for it before completing. + public TaskCompletionSource? ConnectGate { get; set; } + + /// Signalled when is entered. + public TaskCompletionSource ConnectEntered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int DisposeCount => Volatile.Read(ref _disposeCount); + + public int SendsTo(byte unit) => _sendsByUnit.TryGetValue(unit, out var n) ? n : 0; + + public int TotalSends => _sendsByUnit.Values.Sum(); + + public async Task ConnectAsync(CancellationToken ct) + { + ConnectEntered.TrySetResult(); + if (ConnectGate is { } gate) await gate.Task.WaitAsync(ct); + } + + public Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) + { + _sendsByUnit.AddOrUpdate(unitId, 1, (_, n) => n + 1); + var scripted = Handler?.Invoke(unitId, pdu, ct); + return scripted ?? Bank.SendAsync(unitId, pdu, ct); + } + + public ValueTask DisposeAsync() + { + Interlocked.Increment(ref _disposeCount); + return ValueTask.CompletedTask; + } +} From f827b3a1349b31483f23a9dd9f6f1891e1677593 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:19:28 -0400 Subject: [PATCH 04/34] fix(modbus): enforce region/type compatibility at the tag factory P6: Coils/DiscreteInputs carry only Bool and register regions never Bool; the factory (and so the deploy-gate Inspect and the AdminUI parse consult) refuses a mismatch, and an absent dataType defaults per region (Bool for a bit region). A HoldingRegisters+Bool tag used to throw out of the coalescing planner and stall every tag of the subscription; the planner also skips any incompatible pair defensively. The tag editor keeps the type compatible when the region changes; the CLI refuses the pair up front. The array-write parity test records the Coils divergence from DriverArrayWriteSupport (owned outside this lane) as follow-up F1. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../Commands/ReadCommand.cs | 8 +++ .../Commands/WriteCommand.cs | 3 + .../ModbusTagDefinitionFactory.cs | 39 ++++++++++- .../ModbusDriver.cs | 3 + .../TagEditors/ModbusTagConfigEditor.razor | 2 +- .../Uns/TagEditors/ModbusTagConfigModel.cs | 17 ++++- .../ModbusArrayWriteSupportParityTests.cs | 33 +++++++++ .../ModbusRegionTypeValidationTests.cs | 70 +++++++++++++++++++ .../Uns/ModbusTagConfigModelTests.cs | 20 ++++++ 9 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/ReadCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/ReadCommand.cs index a2378a55..3b48f4d1 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/ReadCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/ReadCommand.cs @@ -57,6 +57,14 @@ public sealed class ReadCommand : ModbusCommandBase ValidateEndpoint(); var ct = console.RegisterCancellationHandler(); + // The driver's tag factory refuses an incompatible region/type pair (the tag would read + // BadNodeIdUnknown) — say why up front instead. + if (!ModbusTagDefinitionFactory.IsRegionTypeCompatible(Region, DataType)) + throw new CliFx.Exceptions.CommandException( + Region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs + ? $"Region '{Region}' only carries Bool values (--type Bool); got '{DataType}'." + : $"Region '{Region}' carries register types; use --type BitInRegister for a bit of a register, not '{DataType}'."); + var tagName = SynthesiseTagName(Region, Address, DataType); var tag = new ModbusTagDefinition( Name: tagName, diff --git a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/WriteCommand.cs b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/WriteCommand.cs index 07bb3bf7..e1876a49 100644 --- a/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/WriteCommand.cs +++ b/src/Drivers/Cli/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Cli/Commands/WriteCommand.cs @@ -74,6 +74,9 @@ public sealed class WriteCommand : ModbusCommandBase throw new CliFx.Exceptions.CommandException( $"Region 'Coils' only supports boolean values (--type Bool). " + $"Type '{DataType}' cannot represent a single-bit coil write."); + if (!ModbusTagDefinitionFactory.IsRegionTypeCompatible(Region, DataType)) + throw new CliFx.Exceptions.CommandException( + $"Region '{Region}' carries register types; use --type BitInRegister for a bit of a register, not '{DataType}'."); var tagName = ReadCommand.SynthesiseTagName(Region, Address, DataType); var tag = new ModbusTagDefinition( diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs index 02e2ff34..86b0a4e9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs @@ -50,7 +50,13 @@ public static class ModbusTagDefinitionFactory // Strict enum reads: a present-but-invalid (typo'd) enum rejects the tag // (→ BadNodeIdUnknown) instead of silently defaulting to a wrong-width Good. if (!TagConfigJson.TryReadEnumStrict(root, "region", ModbusRegion.HoldingRegisters, out var region)) return false; - if (!TagConfigJson.TryReadEnumStrict(root, "dataType", ModbusDataType.Int16, out var dataType)) return false; + // An absent dataType defaults per region: a bit region (Coils / DiscreteInputs) only carries + // Bool, so defaulting it to Int16 would reject every bit tag authored without the key. + if (!TagConfigJson.TryReadEnumStrict(root, "dataType", DefaultDataType(region), out var dataType)) return false; + // Region/type compatibility: a bit region carries only Bool, a register region never Bool. + // A mismatch used to throw out of the coalescing planner (killing every tag's poll) or + // silently return a bool into an Int32 node. + if (!IsRegionTypeCompatible(region, dataType)) return false; if (!TagConfigJson.TryReadEnumStrict(root, "byteOrder", ModbusByteOrder.BigEndian, out var byteOrder)) return false; if (!TagConfigJson.TryReadEnumStrict(root, "stringByteOrder", ModbusStringByteOrder.HighByteFirst, out var stringByteOrder)) return false; var bitIndex = (byte)ReadInt(root, "bitIndex"); @@ -94,6 +100,25 @@ public static class ModbusTagDefinitionFactory catch (InvalidOperationException) { return false; } } + /// The data type an authored tag gets when its dataType key is absent. + /// The tag's region. + /// Bool for Coils / DiscreteInputs, Int16 for the register regions. + public static ModbusDataType DefaultDataType(ModbusRegion region) + => region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs ? ModbusDataType.Bool : ModbusDataType.Int16; + + /// + /// Region/type compatibility, the one rule every authoring path and the driver share: Coils and + /// DiscreteInputs carry only Bool; HoldingRegisters and InputRegisters carry every register + /// type (including BitInRegister) but never Bool. + /// + /// The tag's region. + /// The tag's data type. + /// when the driver can read the pair. + public static bool IsRegionTypeCompatible(ModbusRegion region, ModbusDataType dataType) + => region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs + ? dataType == ModbusDataType.Bool + : dataType != ModbusDataType.Bool; + /// /// Inverse of : serialises a typed definition back to the camelCase /// TagConfig JSON the mapper reads. Used by authoring paths (the Client CLI) that construct a @@ -247,7 +272,17 @@ public static class ModbusTagDefinitionFactory // Type-dependent guards. The byte/ushort NARROWING is part of the rule, not an accident of the // mapper: it reads `(byte)bitIndex` / `(ushort)stringLength`, so 256 is bit 0 and is ACCEPTED. // Describing the un-narrowed value here would call a blob invalid that the driver serves. - if (!TagConfigJson.TryReadEnumStrict(root, "dataType", ModbusDataType.Int16, out var dataType)) return; + var region = TagConfigJson.TryReadEnumStrict(root, "region", ModbusRegion.HoldingRegisters, out var r) + ? r : ModbusRegion.HoldingRegisters; + if (!TagConfigJson.TryReadEnumStrict(root, "dataType", DefaultDataType(region), out var dataType)) return; + + if (root.TryGetProperty("region", out _) && !IsRegionTypeCompatible(region, dataType)) + { + warnings.Add(region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs + ? $"Modbus region {region} only carries Bool tags; got {dataType} — the tag is rejected (BadNodeIdUnknown)." + : $"Modbus Bool tags only live in Coils / DiscreteInputs; a {region} tag must use a register type " + + "(use BitInRegister for a bit of a register) — the tag is rejected (BadNodeIdUnknown)."); + } if (dataType == ModbusDataType.String && (ushort)ReadInt(root, "stringLength") < 1) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index 648e4c99..eedfb70b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -1186,6 +1186,9 @@ public sealed class ModbusDriver { if (!table.TryGetValue(fullReferences[i], out var tag)) continue; if (tag.CoalesceProhibited) continue; + // Defensive: the factory refuses incompatible pairs, but the planner's register math must never + // throw for a tag that slipped through — one bad tag would otherwise fail the whole batch. + if (!ModbusTagDefinitionFactory.IsRegionTypeCompatible(tag.Region, tag.DataType)) continue; if (tag.ArrayCount.HasValue) continue; if (tag.Region is not (ModbusRegion.HoldingRegisters or ModbusRegion.InputRegisters)) continue; if (tag.DataType is ModbusDataType.String or ModbusDataType.BitInRegister) continue; diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor index 116f9bd5..a3b38dd4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor @@ -5,7 +5,7 @@
- @foreach (var v in Enum.GetValues()) { }
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs index 323830ab..29050fc4 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs @@ -30,6 +30,19 @@ public sealed class ModbusTagConfigModel /// key stays absent.
 public bool? Writable { get; set; } + /// + /// Changes the region and keeps the data type compatible with it (the driver refuses a bit region + /// with a non-Bool type, and a register region with Bool): switching to Coils / DiscreteInputs selects + /// Bool, switching from them back to a register region selects Int16. + /// + /// The newly selected region. + public void SetRegion(ModbusRegion region) + { + Region = region; + if (!ModbusTagDefinitionFactory.IsRegionTypeCompatible(region, DataType)) + DataType = ModbusTagDefinitionFactory.DefaultDataType(region); + } + private JsonObject _bag = new(); // The blob this model was LOADED FROM, or null when it was built by object initialiser rather than @@ -49,7 +62,9 @@ public sealed class ModbusTagConfigModel _source = json ?? "", Region = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters), Address = TagConfigJson.GetInt(o, "address"), - DataType = TagConfigJson.GetEnum(o, "dataType", ModbusDataType.Int16), + // Absent dataType defaults per region, exactly as the driver's factory does (Bool for a bit region). + DataType = TagConfigJson.GetEnum(o, "dataType", + ModbusTagDefinitionFactory.DefaultDataType(TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters))), ByteOrder = TagConfigJson.GetEnum(o, "byteOrder", ModbusByteOrder.BigEndian), BitIndex = TagConfigJson.GetInt(o, "bitIndex"), StringLength = TagConfigJson.GetInt(o, "stringLength"), diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs index a47a721a..6d2363b1 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs @@ -25,10 +25,43 @@ public sealed class ModbusArrayWriteSupportParityTests return data; } + /// + /// Shapes where the driver has moved ahead of the server-side table (review 07 P6 / U3, Modbus lane) and + /// — owned outside the Modbus lane — has not followed yet + /// (follow-up F1): the tag factory now REFUSES a Coils tag with a non-Bool type (so the driver serves no + /// such tag), and the driver now encodes String[] and BitInRegister[] holding-register writes. + /// fails the moment the table follows, so this list + /// is removed with it rather than left to rot. + /// + private static bool PendingF1(ModbusRegion region, ModbusDataType dataType) => + (region == ModbusRegion.Coils && dataType != ModbusDataType.Bool) + || (region == ModbusRegion.HoldingRegisters && dataType is ModbusDataType.String or ModbusDataType.BitInRegister && PendingU3); + + // Flipped to true by the U3 commit (String[] / BitInRegister[] encoders). + private const bool PendingU3 = false; + + public static TheoryData PendingShapes() + { + var data = new TheoryData(); + foreach (var (region, dt) in Shapes().Where(s => PendingF1(s.Region, s.DataType))) data.Add(region, dt); + return data; + } + + [Theory] + [MemberData(nameof(PendingShapes))] + public void Pending_F1_divergences_still_exist(ModbusRegion region, ModbusDataType dataType) + { + var json = $$"""{"region":"{{region}}","address":10,"dataType":"{{dataType}}","stringLength":4,"isArray":true,"arrayLength":2}"""; + var driverAccepts = region == ModbusRegion.HoldingRegisters; // String[] / BitInRegister[] encode; mistyped Coils are refused + DriverArrayWriteSupport.Supports(DriverTypeNames.Modbus, "Int16", json).ShouldNotBe(driverAccepts, + "F1 landed — DriverArrayWriteSupport now agrees with the driver for this shape; delete it from PendingF1"); + } + [Theory] [MemberData(nameof(AllShapes))] public async Task Table_agrees_with_the_driver_write_path(ModbusRegion region, ModbusDataType dataType) { + if (PendingF1(region, dataType)) return; // covered by Pending_F1_divergences_still_exist var json = $$"""{"region":"{{region}}","address":10,"dataType":"{{dataType}}","stringLength":4,"isArray":true,"arrayLength":2}"""; var fake = new ModbusDriverTests.FakeTransport(); var drv = new ModbusDriver( diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs new file mode 100644 index 00000000..b4acdaee --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs @@ -0,0 +1,70 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Review 07 P6: region/type compatibility is enforced at the tag factory (and so at the deploy gate +/// and the AdminUI parse consult). A HoldingRegisters + Bool tag used to throw out of the +/// coalescing planner and kill the whole subscription's poll cycle; Coils + Int16 silently +/// returned a bool into an Int32 node. +/// +[Trait("Category", "Unit")] +public sealed class ModbusRegionTypeValidationTests +{ + [Theory] + [InlineData("""{"region":"Coils","address":1,"dataType":"Int16"}""")] + [InlineData("""{"region":"DiscreteInputs","address":1,"dataType":"Float32"}""")] + [InlineData("""{"region":"HoldingRegisters","address":1,"dataType":"Bool"}""")] + [InlineData("""{"region":"InputRegisters","address":1,"dataType":"Bool"}""")] + public void Incompatible_region_and_type_is_rejected(string json) + { + ModbusTagDefinitionFactory.FromTagConfig(json, "T", out _).ShouldBeFalse(); + ModbusTagDefinitionFactory.Inspect(json).ShouldContain(w => w.Contains("only") || w.Contains("register")); + } + + [Theory] + [InlineData("""{"region":"Coils","address":1,"dataType":"Bool"}""", ModbusDataType.Bool)] + [InlineData("""{"region":"Coils","address":1}""", ModbusDataType.Bool)] + [InlineData("""{"region":"DiscreteInputs","address":1}""", ModbusDataType.Bool)] + [InlineData("""{"region":"InputRegisters","address":1,"dataType":"BitInRegister","bitIndex":3}""", ModbusDataType.BitInRegister)] + [InlineData("""{"region":"HoldingRegisters","address":1}""", ModbusDataType.Int16)] + public void Compatible_region_and_type_is_accepted(string json, ModbusDataType expected) + { + // Positive control for the rejection theory, including the absent-dataType default per region. + ModbusTagDefinitionFactory.FromTagConfig(json, "T", out var def).ShouldBeTrue(); + def.DataType.ShouldBe(expected); + ModbusTagDefinitionFactory.Inspect(json).ShouldBeEmpty(); + } + + [Fact] + public async Task A_mis_typed_tag_no_longer_kills_the_coalesced_poll_of_its_neighbours() + { + var fake = new ModbusDriverTests.FakeTransport(); + fake.HoldingRegisters[100] = 7; + fake.HoldingRegisters[102] = 9; + var good1 = new ModbusTagDefinition("G1", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16); + var good2 = new ModbusTagDefinition("G2", ModbusRegion.HoldingRegisters, 102, ModbusDataType.Int16); + var opts = new ModbusDriverOptions + { + Host = "f", + MaxReadGap = 5, + RawTags = + [ + ModbusRawTags.Entry(good1), + new RawTagEntry("Bad", """{"region":"HoldingRegisters","address":101,"dataType":"Bool"}""", false), + ModbusRawTags.Entry(good2), + ], + Probe = new ModbusProbeOptions { Enabled = false }, + }; + await using var drv = new ModbusDriver(opts, "m1", _ => fake); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var r = await drv.ReadAsync(["G1", "Bad", "G2"], TestContext.Current.CancellationToken); + r[0].Value.ShouldBe((short)7); + r[1].StatusCode.ShouldBe(0x80340000u); // BadNodeIdUnknown — skipped at init, reported + r[2].Value.ShouldBe((short)9); + drv.GetSkippedTags()!.Sample.ShouldContain(s => s.RawPath == "Bad"); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs index 2bb03383..e1afd0c1 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs @@ -121,4 +121,24 @@ public sealed class ModbusTagConfigModelTests [InlineData("""{"region":3,"address":1,"dataType":true}""")] public void Validate_accepts_every_enum_value_the_driver_accepts(string json) => ModbusTagConfigModel.FromJson(json).Validate().ShouldBeNull(); + + /// Review 07 P6: switching region keeps the data type one the driver accepts for it. + [Theory] + [InlineData(ModbusDataType.Int16, ModbusRegion.Coils, ModbusDataType.Bool)] + [InlineData(ModbusDataType.Float32, ModbusRegion.DiscreteInputs, ModbusDataType.Bool)] + [InlineData(ModbusDataType.Bool, ModbusRegion.HoldingRegisters, ModbusDataType.Int16)] + [InlineData(ModbusDataType.Float32, ModbusRegion.InputRegisters, ModbusDataType.Float32)] + public void SetRegion_keeps_the_type_compatible(ModbusDataType from, ModbusRegion region, ModbusDataType expected) + { + var m = new ModbusTagConfigModel { DataType = from }; + m.SetRegion(region); + m.Region.ShouldBe(region); + m.DataType.ShouldBe(expected); + ModbusTagConfigModel.FromJson(m.ToJson()).Validate().ShouldBeNull(); + } + + /// A Coils + Int16 blob (CSV / API) is refused at authoring, matching the driver. + [Fact] + public void Validate_refuses_an_incompatible_region_and_type() + => ModbusTagConfigModel.FromJson("""{"region":"Coils","address":1,"dataType":"Int16"}""").Validate().ShouldNotBeNull(); } From 01efd3917732eee6a30367369adcf53abb32429c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:22:17 -0400 Subject: [PATCH 05/34] fix(mqtt): retry failed SUBSCRIBEs via a pending set; throw on no-verdict subscribe (R06:M-1, M-11) A failed SUBSCRIBE (refusal or no broker verdict) used to drop its topic from every retry set while SubscribeAsync returned a live handle, leaving tags dark behind a Healthy connection until a redeploy. Failed topics now join a pending set retried by the next SubscribeAsync and by every reconnect (not counted toward the reconnect's total-failure verdict, so an ACL-denied topic cannot reconnect-loop). A no-verdict subscribe degrades its refs then throws so the actor's reconcile re-subscribes. A subscribe with no transport attached is recorded pending, never established (M-11). MiniBroker now answers SUBSCRIBE. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 15 +- .../MqttSubscriptionManager.cs | 131 ++++++++++---- .../MiniBroker.cs | 52 ++++++ .../MqttDriverSessionLifecycleTests.cs | 53 +++++- .../MqttSubscriptionManagerTests.cs | 163 +++++++++++++++++- 5 files changed, 374 insertions(+), 40 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index f8aef796..8400ab42 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -408,9 +408,18 @@ value. `MqttConnection` therefore raises `MqttConnectRejectedException`: code. (`ServerMoved` is permanent, `UseAnotherServer` is temporary — per the MQTT 5 spec.) A reconnect that completes without re-subscribing would be a healthy-looking connection that receives -nothing forever, so `OnReconnectedAsync` **throws** when zero filters are granted, tearing the session -down rather than swallowing it. A SUBACK failure degrades just the affected RawPaths to -`BadCommunicationError`. +nothing forever, so `OnReconnectedAsync` **throws** when none of the previously live filters is granted +(or the SUBSCRIBE gets no verdict), tearing the session down rather than swallowing it. A SUBACK failure +degrades just the affected RawPaths to `BadCommunicationError`. + +**A failed SUBSCRIBE is retried, never forgotten (R06:M-1).** Every topic whose SUBSCRIBE was refused or got +no verdict joins a *pending* set that the next `SubscribeAsync` and every reconnect re-request; a grant moves +it to the live set, and a redeploy that drops its tags drops it. Pending topics do not count toward the +reconnect's total-failure verdict, so one ACL-denied topic cannot reconnect-loop the driver. A plain-mode +`SubscribeAsync` whose SUBSCRIBE got **no verdict at all** (transport throw, SUBSCRIBE deadline) degrades its +refs and then **throws**, so `DriverInstanceActor` holds no handle and its health-tick reconcile re-subscribes; +a SUBACK *refusal* still returns a handle (the broker answered). A subscribe issued with no transport attached +records its topics as pending, never as established (M-11). There are **two** kinds of reconnect and only one of them fires that callback. A drop the connection's own supervisor recovers keeps the same `MqttConnection`, so `Reconnected` → `OnReconnectedAsync` diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 92c45051..9e4d8f64 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -113,6 +113,14 @@ public sealed class MqttSubscriptionManager /// Topics already SUBSCRIBEd on the live session — the dedupe set and the re-subscribe set. private readonly ConcurrentDictionary _subscribedTopics = new(StringComparer.Ordinal); + /// + /// Topics a subscribe asked for whose SUBSCRIBE failed (refused, or no verdict at all). Retried by + /// every later and by ; moved to + /// when granted. Without it a failed topic left every retry set and + /// stayed dark behind a Healthy connection until a redeploy. + /// + private readonly ConcurrentDictionary _pendingTopics = new(StringComparer.Ordinal); + /// RawPaths whose extraction failure has already been logged loudly once. private readonly ConcurrentDictionary _warnedRefs = new(StringComparer.Ordinal); @@ -212,8 +220,8 @@ public sealed class MqttSubscriptionManager /// The manager does not own the connection's lifetime and never detaches — the driver disposes /// both together. Call once per connection, and before the first /// : a subscribe issued with no transport attached records its - /// intent (and warns) but sends nothing to a broker, and would only be established if the - /// connection later dropped and recovered. + /// topics as pending (and warns) but sends nothing to a broker; the next subscribe or + /// reconnect on an attached session retries them. /// /// /// Binding to a session forgets the previous session's established filters — the same @@ -328,6 +336,14 @@ public sealed class MqttSubscriptionManager } } + foreach (var topic in _pendingTopics.Keys) + { + if (!table.ByFilter.ContainsKey(topic)) + { + _pendingTopics.TryRemove(topic, out _); + } + } + foreach (var rawPath in _handleByRawPath.Keys) { if (!table.ByRawPath.ContainsKey(rawPath)) @@ -356,12 +372,13 @@ public sealed class MqttSubscriptionManager /// opening SUBSCRIBE itself; only re-establishes it later. /// /// - /// Never throws, never hangs. A reference that is not an authored tag is recorded as - /// BadNodeIdUnknown; a SUBACK rejection, transport failure or subscribe deadline - /// degrades exactly the affected references to BadCommunicationError. Either way a - /// handle is returned — the OPC UA server's subscribe call must not fail because one topic - /// is ACL-denied. Recovery for a transient failure comes from the reconnect path; - /// a persistent broker-side refusal is visible as a Bad status on the affected nodes. + /// Never hangs; throws only when the broker gave no verdict. A reference that is not an + /// authored tag is recorded as BadNodeIdUnknown. A SUBACK rejection degrades exactly the + /// affected references to BadCommunicationError and still returns a handle — the OPC UA + /// server's subscribe must not fail because one topic is ACL-denied. A transport failure or + /// subscribe deadline degrades the same way and then throws, so the host holds no handle + /// and its reconcile re-subscribes. Every failed topic also joins a pending set that the next + /// subscribe and every reconnect retry. /// /// /// The RawPath references to subscribe. @@ -414,12 +431,43 @@ public sealed class MqttSubscriptionManager // One SUBSCRIBE per distinct topic, and only for topics not already live: re-SUBSCRIBEing an // established filter is legal but pointless, and it re-triggers the retained seed. - var pending = defs.Where(d => !_subscribedTopics.ContainsKey(d.Topic)).ToList(); - await EstablishAsync(BuildFilters(pending, _defaultQos), cancellationToken).ConfigureAwait(false); + // Topics an EARLIER subscribe failed to establish ride along: a redeploy's bulk re-subscribe is then a + // retry of a refusal the broker may since have lifted, instead of a no-op that leaves them dark. + var authored = _authored; + var pending = defs.Where(d => !_subscribedTopics.ContainsKey(d.Topic)) + .Concat(PendingDefinitions(authored)) + .ToList(); + var outcome = await EstablishAsync(BuildFilters(pending, _defaultQos), cancellationToken).ConfigureAwait(false); + if (outcome.TransportFailure is { } failure) + { + // No verdict at all (transport throw / SUBSCRIBE deadline): THROW, so the caller holds no handle + // and DriverInstanceActor's health-tick reconcile re-subscribes. Returning a handle here was the + // silent-death case — the actor thought the subscription live and never retried. The refs already + // carry BadCommunicationError (raised under this handle above), so the throw costs no visibility. + _refsByHandle.TryRemove(handle, out _); + foreach (var def in defs) + { + _handleByRawPath.TryRemove(new KeyValuePair(def.Name, handle)); + } + + throw new InvalidOperationException( + $"MQTT driver '{_driverId}': SUBSCRIBE for {pending.Count} tag(s) got no broker verdict " + + $"({failure.GetType().Name}); the subscription will be retried.", + failure); + } + + // A SUBACK refusal (an ACL denial, say) does NOT throw: the broker answered, the refused topics are + // Bad + pending, and failing the whole subscribe would take every other topic in it dark too. return handle; } + /// The authored definitions behind every still-authored pending topic. + private IEnumerable PendingDefinitions(AuthoredTable authored) + => _pendingTopics.Keys + .Select(topic => authored.ByFilter.TryGetValue(topic, out var defs) ? defs : []) + .SelectMany(defs => defs); + /// /// Stops attributing notifications to . References another live /// subscription also covers keep publishing. @@ -498,8 +546,13 @@ public sealed class MqttSubscriptionManager .SelectMany(defs => defs) .ToList(); - var filters = BuildFilters(live, _defaultQos); - if (filters.Count == 0) + // Pending topics (a failed earlier SUBSCRIBE) are retried here too — a reconnect re-subscribing + // only the granted set is how a failed topic stayed dark forever. They do NOT count toward the + // total-failure verdict below: re-refusing a topic that was never live loses nothing, and throwing + // for it would turn one ACL-denied topic into a reconnect loop for the whole driver. + var pendingDefs = PendingDefinitions(authored).ToList(); + var filters = BuildFilters(live.Concat(pendingDefs), _defaultQos); + if (live.Count == 0 && pendingDefs.Count == 0) { // Silence here is how the wildcard defect hid: no exception, no log, no SUBSCRIBE. Live // topics resolving to zero filters re-establishes nothing, which is the total-failure case @@ -519,13 +572,17 @@ public sealed class MqttSubscriptionManager return; } - var granted = await EstablishAsync(filters, cancellationToken).ConfigureAwait(false); - if (granted == 0) + var outcome = await EstablishAsync(filters, cancellationToken).ConfigureAwait(false); + var liveTopics = live.Select(d => d.Topic).Distinct(StringComparer.Ordinal).ToList(); + var liveLost = outcome.TransportFailure is not null + || (subscribed.Count > 0 && (liveTopics.Count == 0 || !liveTopics.Any(outcome.Granted.Contains))); + if (liveLost) { throw new InvalidOperationException( $"MQTT driver '{_driverId}': no subscription could be re-established after reconnect " + $"({filters.Count} filter(s) requested); tearing the session down rather than serving a " - + "connection that receives nothing."); + + "connection that receives nothing.", + outcome.TransportFailure); } } @@ -633,35 +690,35 @@ public sealed class MqttSubscriptionManager /// /// The filters to establish; an empty list is a no-op. /// Cancellation for the round-trip. - /// How many filters the broker granted. - private async Task EstablishAsync( + /// The granted topics, plus the transport failure when the broker gave no verdict at all. + private async Task EstablishAsync( IReadOnlyList filters, CancellationToken cancellationToken) { if (filters.Count == 0) { - return 0; + return EstablishOutcome.Nothing; } var transport = _transport; if (transport is null) { - // No transport: the intent is recorded so the reconnect path establishes it, but NOTHING - // reaches a broker now. Loud, because outside tests this means the composition order is - // wrong — AttachTo must run before the OPC UA server's first subscribe, or these topics - // stay unestablished until the connection happens to drop and recover. + // No transport: NOTHING reaches a broker, so nothing is recorded as subscribed — recording it + // was a silent "subscribed, receives nothing" if the AttachTo-before-subscribe ordering ever + // changed. The topics wait in the pending set, which the next subscribe and every reconnect + // retry. Loud, because outside tests this means the composition order is wrong. foreach (var filter in filters) { - _subscribedTopics[filter.Topic] = 0; + _pendingTopics[filter.Topic] = 0; } _logger?.LogWarning( - "MQTT driver '{DriverId}': {FilterCount} subscription(s) recorded with no connection attached; " - + "nothing was sent to a broker. Call AttachTo before subscribing.", + "MQTT driver '{DriverId}': {FilterCount} subscription(s) requested with no connection attached; " + + "nothing was sent to a broker and they stay pending. Call AttachTo before subscribing.", _driverId, filters.Count); - return filters.Count; + return EstablishOutcome.Nothing; } IReadOnlyList outcomes; @@ -681,16 +738,17 @@ public sealed class MqttSubscriptionManager DegradeTopic(filter.Topic, ex.GetType().Name); } - return 0; + return new EstablishOutcome(new HashSet(StringComparer.Ordinal), ex); } - var granted = 0; + var granted = new HashSet(StringComparer.Ordinal); foreach (var outcome in outcomes) { if (outcome.Granted) { _subscribedTopics[outcome.Topic] = 0; - granted++; + _pendingTopics.TryRemove(outcome.Topic, out _); + granted.Add(outcome.Topic); } else { @@ -703,7 +761,16 @@ public sealed class MqttSubscriptionManager } } - return granted; + return new EstablishOutcome(granted, null); + } + + /// One SUBSCRIBE round-trip's verdict. + /// The filters the broker granted. + /// Set when the call produced no verdict at all (throw / deadline). + private sealed record EstablishOutcome(IReadOnlySet Granted, Exception? TransportFailure) + { + public static readonly EstablishOutcome Nothing = + new(new HashSet(StringComparer.Ordinal), null); } /// Marks every authored reference on as unreachable. @@ -722,6 +789,7 @@ public sealed class MqttSubscriptionManager if (!_authored.ByFilter.TryGetValue(topic, out var defs)) { + _pendingTopics.TryRemove(topic, out _); _logger?.LogWarning( "MQTT driver '{DriverId}': cannot degrade tags for unestablished filter '{Topic}' ({Reason}) — " + "no authored tag resolves to it.", @@ -731,6 +799,9 @@ public sealed class MqttSubscriptionManager return; } + // Still authored, so it must be retried — see _pendingTopics. + _pendingTopics[topic] = 0; + var now = DateTime.UtcNow; var snapshot = new DataValueSnapshot(null, StatusBadCommunicationError, null, now); foreach (var def in defs) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs index 7950a240..77d0b762 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs @@ -59,6 +59,54 @@ internal sealed class MiniBroker : IDisposable /// public byte ConnAckReturnCode { get; set; } = ReturnCodeAccepted; + /// + /// When true (the default) a SUBSCRIBE is answered with a SUBACK granting each filter's requested + /// QoS. False is a broker that accepts SUBSCRIBE and never answers — the subscribe-deadline shape. + /// + public bool AnswerSubscribe { get; set; } = true; + + /// + /// Builds the SUBACK for one SUBSCRIBE packet read whole into , or null when + /// the packet is not parseable from one read (the fake handles one packet per read, like CONNECT). + /// + private static byte[]? TryBuildSubAck(byte[] buffer, int read) + { + // Fixed header: type byte, then the variable-length "remaining length". + var at = 1; + var remaining = 0; + var multiplier = 1; + byte digit; + do + { + if (at >= read) return null; + digit = buffer[at++]; + remaining += (digit & 0x7F) * multiplier; + multiplier *= 128; + } + while ((digit & 0x80) != 0); + + var end = at + remaining; + if (end > read || remaining < 2) return null; + + var packetIdHigh = buffer[at]; + var packetIdLow = buffer[at + 1]; + at += 2; + + var grants = new List(); + while (at + 2 <= end) + { + var length = (buffer[at] << 8) | buffer[at + 1]; + at += 2 + length; + if (at >= end) return null; + grants.Add((byte)(buffer[at] & 0x03)); + at++; + } + + var ack = new List { 0x90, (byte)(2 + grants.Count), packetIdHigh, packetIdLow }; + ack.AddRange(grants); + return [.. ack]; + } + /// Total TCP connections accepted since construction — counts reconnect attempts. public int AcceptedCount => Volatile.Read(ref _acceptedCount); @@ -163,6 +211,10 @@ internal sealed class MiniBroker : IDisposable { await stream.WriteAsync(PingResp, _cts.Token); } + else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(buffer, read) is { } subAck) + { + await stream.WriteAsync(subAck, _cts.Token); + } } } catch (Exception) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index ca4e667e..947fb081 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -13,9 +13,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; /// ( / ). /// /// -/// Plain mode deliberately: it is the only mode whose initialize needs no SUBSCRIBE, and -/// answers CONNECT and PINGREQ and nothing else — so a Sparkplug driver -/// would park on its group SUBSCRIBE until the deadline and prove nothing about connections. +/// Plain mode deliberately: it is the only mode whose initialize needs no SUBSCRIBE. +/// answers CONNECT, SUBSCRIBE (unless told not to) and PINGREQ, and never +/// PUBLISHes, so ingest is driven through Subscriptions.HandleMessage. /// [Trait("Category", "Unit")] public sealed class MqttDriverSessionLifecycleTests @@ -122,7 +122,7 @@ public sealed class MqttDriverSessionLifecycleTests /// /// /// Ingest is fed through Subscriptions.HandleMessage, the method MQTTnet's dispatcher - /// calls, because answers CONNECT and PINGREQ and nothing else: it + /// calls, because answers CONNECT, SUBSCRIBE and PINGREQ only: it /// can drop a session (which is what this test needs from it) but cannot PUBLISH. /// /// @@ -174,6 +174,51 @@ public sealed class MqttDriverSessionLifecycleTests recovered.Value.ShouldBe(21.5); } + /// + /// R06:M-1, end to end. A SUBSCRIBE the broker never answers used to come back from + /// as a live handle while the topic left every retry set, so + /// DriverInstanceActor never re-subscribed and the tag stayed dark behind a Healthy connection. + /// It must throw (the actor's reconcile then retries), and the retry against a broker that answers + /// must establish and deliver — the positive control. + /// + [Trait("Category", "SlowUnit")] + [Fact] + public async Task ASubscribeTheBrokerNeverAnswers_Throws_AndTheRetryEstablishes() + { + const string rawPath = "Plant/Mqtt/dev1/Temp"; + const string topic = "factory/oven/temp"; + + using var broker = new MiniBroker { AnswerSubscribe = false }; + var options = BrokerOptions(broker.Port) with + { + RawTags = + [ + new RawTagEntry( + rawPath, + $$"""{"topic":"{{topic}}","payloadFormat":"Scalar","dataType":"Float64"}""", + WriteIdempotent: false), + ], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + + await Should.ThrowAsync(() => + driver.SubscribeAsync([rawPath], TimeSpan.Zero, TestContext.Current.CancellationToken)); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "the connection itself is fine — that was the trap"); + + broker.AnswerSubscribe = true; + var handle = await driver.SubscribeAsync([rawPath], TimeSpan.Zero, TestContext.Current.CancellationToken); + + var raised = new List(); + driver.OnDataChange += (_, e) => { lock (raised) raised.Add(e); }; + driver.Subscriptions.HandleMessage(topic, "7"u8.ToArray(), retained: false); + lock (raised) + { + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.StatusCode.ShouldBe(0u); + } + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs index 558d4477..632b0925 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs @@ -538,17 +538,171 @@ public sealed class MqttSubscriptionManagerTests mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); } + /// + /// A SUBSCRIBE that never got a verdict (transport throw / deadline) degrades its refs AND throws, + /// so DriverInstanceActor holds no handle and its health-tick reconcile re-subscribes. It + /// used to return a live-looking handle with the topic dropped from every retry set — dark behind a + /// Healthy connection until a redeploy. + /// [Fact] - public async Task SubscribeAsync_TransportThrows_DegradesRefsToBad_DoesNotThrow() + public async Task SubscribeAsync_TransportThrows_DegradesRefsToBad_AndThrows() { var transport = new FakeTransport { Throw = new InvalidOperationException("broker gone") }; var mgr = NewManager(transport); mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); - await Should.NotThrowAsync(() => + await Should.ThrowAsync(() => mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); + // The Bad verdict still reached the subscriber before the throw. + raised.ShouldContain(e => e.FullReference == OvenTempPath && e.Snapshot.StatusCode == BadCommunicationError); + } + + /// + /// The reconcile path: once the actor re-subscribes after the throw, the topic IS re-SUBSCRIBEd and + /// delivery resumes on the new handle — the positive control for the throw above. + /// + [Fact] + public async Task SubscribeAsync_AfterATransportThrow_TheRetrySubscribesAndDelivers() + { + var transport = new FakeTransport { Throw = new InvalidOperationException("broker gone") }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await Should.ThrowAsync(() => + mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); + + transport.Throw = null; + var handle = await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); + mgr.HandleMessage(OvenTopic, "21.5"u8.ToArray(), retained: false); + + transport.Calls.Count.ShouldBe(2); + transport.Calls[1].Select(f => f.Topic).ShouldBe([OvenTopic]); + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.Value.ShouldBe(21.5); + } + + /// + /// A topic whose first SUBSCRIBE was refused stays in a pending set the reconnect re-subscribe + /// covers. The reconnect used to re-SUBSCRIBE only the granted set, so it was never retried. + /// + [Fact] + public async Task OnReconnectedAsync_RetriesATopicWhoseFirstSubscribeWasRefused() + { + var transport = new FakeTransport { Reject = _ => true }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + var handle = await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Reject = _ => false; + await mgr.OnReconnectedAsync(TestContext.Current.CancellationToken); + + transport.Calls.Count.ShouldBe(2); + transport.Calls[1].Select(f => f.Topic).ShouldBe([OvenTopic]); + + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); + mgr.HandleMessage(OvenTopic, "3"u8.ToArray(), retained: false); + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.StatusCode.ShouldBe(Good); + } + + /// + /// A persistently ACL-denied topic must not turn every reconnect into a teardown loop: a pending + /// topic refused again is not "re-established nothing" — nothing that was live was lost. Both a + /// mixed set (the granted topic still re-subscribes) and a pending-only set are covered. + /// + [Fact] + public async Task OnReconnectedAsync_PendingTopicsRefusedAgain_DoesNotThrow() + { + var transport = new FakeTransport { Reject = t => t == "f/oven/press" }; + var mgr = NewManager(transport); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenTempPath, OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + await Should.NotThrowAsync(() => mgr.OnReconnectedAsync(TestContext.Current.CancellationToken)); + transport.Calls[1].Select(f => f.Topic).OrderBy(t => t, StringComparer.Ordinal) + .ShouldBe(["f/oven/press", OvenTopic]); + + var lone = new FakeTransport { Reject = _ => true }; + var mgr2 = NewManager(lone); + mgr2.Register([Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr2.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + await Should.NotThrowAsync(() => mgr2.OnReconnectedAsync(TestContext.Current.CancellationToken)); + lone.Calls.Count.ShouldBe(2); // it WAS retried + } + + /// + /// A later subscribe on the same session (a redeploy's bulk re-subscribe) also retries the pending + /// set, so a broker ACL fixed while connected recovers without waiting for a reconnect. + /// + [Fact] + public async Task SubscribeAsync_RetriesPendingTopicsFromAnEarlierRefusal() + { + var transport = new FakeTransport { Reject = t => t == "f/oven/press" }; + var mgr = NewManager(transport); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Reject = _ => false; + await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Calls[1].Select(f => f.Topic).OrderBy(t => t, StringComparer.Ordinal) + .ShouldBe(["f/oven/press", OvenTopic]); + } + + /// + /// R06:M-11. A subscribe issued before any transport is attached reaches no broker, so it must + /// not be recorded as established. It used to be — "subscribed, receives nothing" — and the first + /// reconnect then had nothing to re-establish. Now it stays pending, and the first reconnect on the + /// attached session attempts it: the Bad verdict from the unconnected session is the proof. + /// + [Fact] + public async Task SubscribeAsync_WithNoTransport_StaysPending_AndTheFirstReconnectAttemptsIt() + { + var mgr = NewManager(transport: null); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(0x80320000u); // BadWaitingForInitialData + + await using var attached = new MqttConnection( + new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false }, "mqtt-1"); + mgr.AttachTo(attached); + + // The unconnected session gives no verdict, so the reconnect throws — and it ATTEMPTED the topic. + await Should.ThrowAsync(() => mgr.OnReconnectedAsync(TestContext.Current.CancellationToken)); + mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); + } + + /// + /// Register drops a pending topic whose tags the redeploy removed — the same housekeeping the + /// granted set gets — so a deleted tag's denied topic is not retried forever. + /// + [Fact] + public async Task Register_DroppingATag_RemovesItsTopicFromThePendingSet() + { + var transport = new FakeTransport { Reject = _ => true }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.OnReconnectedAsync(TestContext.Current.CancellationToken); + + transport.Calls.Count.ShouldBe(1); // nothing left to retry } // --------------------------------------------------------------------------------- @@ -939,7 +1093,10 @@ public sealed class MqttSubscriptionManagerTests new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false }, "mqtt-1"); mgr.AttachTo(rebound); - await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + // The unconnected session gives no verdict, so the subscribe now THROWS (M-1) — the throw is a + // second proof the SUBSCRIBE was attempted against the new session. + await Should.ThrowAsync(() => + mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe( BadCommunicationError, From 2ef3cd0df25f80f6a18297f6aaea8cc2489b48ab Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:24:38 -0400 Subject: [PATCH 06/34] feat(modbus): block-read coalescing on by default; coils/DI/BitInRegister/String fold U2: MaxReadGap defaults to 8 and 0 now means adjacent-only rather than off; a new DisableReadCoalescing switch (factory DTO + AdminUI form) turns it off entirely. The planner also merges Coils/DiscreteInputs (FC01/FC02, gap in bits), String scalars and BitInRegister tags - sixteen bit tags on one status word are one FC03 instead of sixteen. Auto-prohibition and its re-probe / bisection use the region's own function code (a coil range was re-probed with FC04). Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriverOptions.cs | 25 ++- .../ModbusDriver.cs | 113 ++++++++----- .../ModbusDriverFactoryExtensions.cs | 5 +- .../Drivers/Forms/ModbusDriverForm.razor | 13 +- .../ModbusCoalescingDefaultsTests.cs | 155 ++++++++++++++++++ .../ModbusCoalescingTests.cs | 6 +- .../ScriptedTransport.cs | 8 + 7 files changed, 269 insertions(+), 56 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs index 39a4104e..9d6b5479 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs @@ -107,14 +107,25 @@ public sealed class ModbusDriverOptions public TimeSpan? AutoProhibitReprobeInterval { get; init; } = null; /// - /// Block-read coalescing budget. When non-zero, the read planner combines tags - /// in the same (UnitId, Region) group whose addresses are at most this many registers - /// apart into a single FC03/FC04/FC01/FC02 read. The sliced response is then dispatched - /// back to per-tag values. Default 0 = no coalescing — every tag gets its own - /// PDU (preserves the previous behaviour). Typical opt-in values are 5..32 — large enough - /// to bridge a few unused registers, small enough to avoid trampling protected holes. + /// Block-read coalescing budget. The read planner combines tags in the same (UnitId, Region) + /// group whose addresses are at most this many registers (Coils / DiscreteInputs: bits) apart + /// into a single FC03/FC04/FC01/FC02 read, then slices the response back into per-tag values. + /// 0 coalesces only adjacent or overlapping tags (several BitInRegister tags on one + /// status word still fold into one read). Default 8. A range the device refuses is + /// auto-prohibited and read per tag, so a protected hole costs one extra round trip, not data. To + /// turn coalescing off entirely set , or exempt one tag with + /// its coalesceProhibited key. /// - public ushort MaxReadGap { get; init; } = 0; + public ushort MaxReadGap { get; init; } = DefaultMaxReadGap; + + /// The default (shared by the factory DTO and the AdminUI form). + public const ushort DefaultMaxReadGap = 8; + + /// + /// When true, every tag is read with its own PDU (no block coalescing at all) — for a device + /// that mis-answers merged reads in a way the auto-prohibition cannot catch. Default false. + /// + public bool DisableReadCoalescing { get; init; } = false; /// /// PLC family hint that drives the parser's family-native branch. When set to a diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index eedfb70b..cc252539 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -727,7 +727,7 @@ public sealed class ModbusDriver var results = new DataValueSnapshot[fullReferences.Count]; var batch = new BatchOutcome(); - var coalesced = _options.MaxReadGap > 0 + var coalesced = !_options.DisableReadCoalescing ? await ReadCoalescedAsync(transport, table, fullReferences, results, now, batch, cancellationToken).ConfigureAwait(false) : new HashSet(); @@ -1083,11 +1083,10 @@ public sealed class ModbusDriver private async Task StraightReprobeAsync(IModbusTransport transport, (byte Unit, ModbusRegion Region, ushort Start, ushort End) key, CancellationToken ct) { - var fc = key.Region == ModbusRegion.HoldingRegisters ? (byte)0x03 : (byte)0x04; var qty = (ushort)(key.End - key.Start + 1); try { - _ = await ReadRegisterBlockAsync(transport, key.Unit, fc, key.Start, qty, ct).ConfigureAwait(false); + _ = await ReadRangeAsync(transport, key.Unit, key.Region, key.Start, qty, ct).ConfigureAwait(false); lock (_autoProhibitedLock) _autoProhibited.Remove(key); LogProhibitionCleared(key.Unit, key.Region, key.Start, key.End); } @@ -1107,13 +1106,12 @@ public sealed class ModbusDriver private async Task BisectAndReprobeAsync(IModbusTransport transport, (byte Unit, ModbusRegion Region, ushort Start, ushort End) key, CancellationToken ct) { - var fc = key.Region == ModbusRegion.HoldingRegisters ? (byte)0x03 : (byte)0x04; var mid = (ushort)((key.Start + key.End) / 2); var leftEnd = mid; var rightStart = (ushort)(mid + 1); - var leftFailed = await ProbeFailsAsync(transport, fc, key.Unit, key.Start, leftEnd, ct).ConfigureAwait(false); - var rightFailed = await ProbeFailsAsync(transport, fc, key.Unit, rightStart, key.End, ct).ConfigureAwait(false); + var leftFailed = await ProbeFailsAsync(transport, key.Region, key.Unit, key.Start, leftEnd, ct).ConfigureAwait(false); + var rightFailed = await ProbeFailsAsync(transport, key.Region, key.Unit, rightStart, key.End, ct).ConfigureAwait(false); lock (_autoProhibitedLock) { @@ -1149,13 +1147,13 @@ public sealed class ModbusDriver } } - private async Task ProbeFailsAsync(IModbusTransport transport, byte fc, byte unit, + private async Task ProbeFailsAsync(IModbusTransport transport, ModbusRegion region, byte unit, ushort start, ushort end, CancellationToken ct) { var qty = (ushort)(end - start + 1); try { - _ = await ReadRegisterBlockAsync(transport, unit, fc, start, qty, ct).ConfigureAwait(false); + _ = await ReadRangeAsync(transport, unit, region, start, qty, ct).ConfigureAwait(false); return false; } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } @@ -1163,12 +1161,15 @@ public sealed class ModbusDriver } /// - /// Block-read coalescing planner. Groups eligible tags by (UnitId, Region), sorts - /// by start address, and merges adjacent / near-adjacent (gap ≤ MaxReadGap) into single - /// FC03/FC04 reads. Per-block: emit one Modbus PDU, slice the response back into per-tag - /// values, populate and the WriteOnChangeOnly cache. Returns - /// the set of indices the planner handled — the - /// caller falls back to the per-tag path for the rest (arrays, coils, prohibited, unknown). + /// Block-read coalescing planner. Groups eligible tags by (UnitId, Region), sorts by start address, + /// and merges adjacent / near-adjacent tags (gap ≤ , 0 = + /// adjacent or overlapping only) into one read per block: FC03/FC04 for the register regions (scalar + /// numeric, String and BitInRegister tags — several bit tags on ONE status word fold into + /// a single read of it), FC01/FC02 for Coils / DiscreteInputs (gap counted in bits). Per block: emit + /// one PDU, slice the response back into per-tag values, populate and the + /// WriteOnChangeOnly cache. Returns the set of indices the planner + /// handled — the caller falls back to the per-tag path for the rest (arrays, lone tags, prohibited, + /// unknown, refused ranges). /// private async Task> ReadCoalescedAsync( IModbusTransport transport, @@ -1179,9 +1180,8 @@ public sealed class ModbusDriver BatchOutcome batch, CancellationToken ct) { - // Eligible: known tag, register region (HoldingRegisters or InputRegisters), scalar - // (no array, no string), not CoalesceProhibited, not BitInRegister. - var eligible = new List<(int Index, string Ref, ModbusTagDefinition Tag)>(); + // Eligible: known scalar tag, not CoalesceProhibited, with a region/type pair the codec can slice. + var eligible = new List<(int Index, ModbusTagDefinition Tag)>(); for (var i = 0; i < fullReferences.Count; i++) { if (!table.TryGetValue(fullReferences[i], out var tag)) continue; @@ -1190,9 +1190,7 @@ public sealed class ModbusDriver // throw for a tag that slipped through — one bad tag would otherwise fail the whole batch. if (!ModbusTagDefinitionFactory.IsRegionTypeCompatible(tag.Region, tag.DataType)) continue; if (tag.ArrayCount.HasValue) continue; - if (tag.Region is not (ModbusRegion.HoldingRegisters or ModbusRegion.InputRegisters)) continue; - if (tag.DataType is ModbusDataType.String or ModbusDataType.BitInRegister) continue; - eligible.Add((i, fullReferences[i], tag)); + eligible.Add((i, tag)); } if (eligible.Count == 0) return new HashSet(); @@ -1201,42 +1199,43 @@ public sealed class ModbusDriver // Group by (UnitId, Region) — coalescing across slaves or regions is unsafe. foreach (var group in eligible.GroupBy(e => (Unit: ResolveUnitId(e.Tag), e.Tag.Region))) { - var fc = group.Key.Region == ModbusRegion.HoldingRegisters ? (byte)0x03 : (byte)0x04; - var cap = _options.MaxRegistersPerRead == 0 ? (ushort)125 : _options.MaxRegistersPerRead; + var isBits = IsBitRegion(group.Key.Region); + var cap = isBits + ? (_options.MaxCoilsPerRead == 0 ? 2000 : _options.MaxCoilsPerRead) + : (_options.MaxRegistersPerRead == 0 ? 125 : _options.MaxRegistersPerRead); var sorted = group.OrderBy(e => e.Tag.Address).ToList(); - // Build merged blocks. A "block" is (start, lastEnd, members[]) where lastEnd is - // the inclusive end address of the last tag's register span. A new tag joins the - // block if its start ≤ lastEnd + 1 + MaxReadGap AND the resulting span ≤ cap. - var blocks = new List<(ushort Start, ushort End, List<(int Index, ModbusTagDefinition Tag)> Members)>(); - foreach (var (idx, _, tag) in sorted) + // Build merged blocks. A "block" is (start, lastEnd, members[]) where lastEnd is the inclusive + // end of the widest member span seen so far. A new tag joins the block if its start ≤ lastEnd + + // 1 + MaxReadGap AND the resulting span ≤ cap AND the span crosses no auto-prohibited range. + var blocks = new List<(int Start, int End, List<(int Index, ModbusTagDefinition Tag)> Members)>(); + foreach (var (idx, tag) in sorted) { - var tagStart = tag.Address; - var tagEnd = (ushort)(tag.Address + RegisterCount(tag) - 1); + var tagStart = (int)tag.Address; + var tagEnd = tagStart + (isBits ? 1 : RegisterCount(tag)) - 1; + if (tagEnd > ushort.MaxValue || tagEnd - tagStart + 1 > cap) continue; // per-tag path chunks / refuses it if (blocks.Count > 0) { var last = blocks[^1]; var gap = tagStart - last.End - 1; var newEnd = Math.Max(tagEnd, last.End); var newSpan = newEnd - last.Start + 1; - var crossesProhibition = RangeIsAutoProhibited(group.Key.Unit, group.Key.Region, last.Start, (ushort)newEnd); + var crossesProhibition = RangeIsAutoProhibited(group.Key.Unit, group.Key.Region, (ushort)last.Start, (ushort)newEnd); if (gap <= _options.MaxReadGap && newSpan <= cap && !crossesProhibition) { last.Members.Add((idx, tag)); - blocks[^1] = (last.Start, (ushort)newEnd, last.Members); + blocks[^1] = (last.Start, newEnd, last.Members); continue; } } blocks.Add((tagStart, tagEnd, new List<(int, ModbusTagDefinition)> { (idx, tag) })); } - // Issue one PDU per block. On a Modbus-level exception (illegal data address / - // protected register), record the range as auto-prohibited, leave the - // member indices UNhandled, and let the per-tag fallback in ReadAsync read each - // surviving address individually. On transport-level failure (timeout / socket - // drop) mark members Bad and short-circuit the per-tag fallback (hitting the - // dead socket again won't help). Bisection narrows the prohibition over - // subsequent re-probe ticks. + // Issue one PDU per block. On a device REFUSAL (illegal data address / protected register), + // record the range as auto-prohibited, leave the member indices UNhandled, and let the per-tag + // fallback in ReadAsync read each surviving address individually. On a transport-level failure + // (timeout / socket drop) mark members Bad and short-circuit the per-tag fallback (hitting the + // dead socket again won't help). Bisection narrows the prohibition over subsequent re-probe ticks. foreach (var block in blocks) { if (block.Members.Count == 1) @@ -1246,6 +1245,7 @@ public sealed class ModbusDriver continue; } + var start = (ushort)block.Start; var qty = (ushort)(block.End - block.Start + 1); if (batch.IsFailedFast(group.Key.Unit)) { @@ -1259,7 +1259,7 @@ public sealed class ModbusDriver byte[] data; try { - data = await ReadRegisterBlockAsync(transport, group.Key.Unit, fc, block.Start, qty, ct).ConfigureAwait(false); + data = await ReadRangeAsync(transport, group.Key.Unit, group.Key.Region, start, qty, ct).ConfigureAwait(false); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -1271,7 +1271,7 @@ public sealed class ModbusDriver // as auto-prohibited and leave the members UNhandled so the per-tag fallback reads each // surviving address individually. The device answered, so health is not degraded. batch.Answered = true; - RecordAutoProhibition(group.Key.Unit, group.Key.Region, block.Start, block.End); + RecordAutoProhibition(group.Key.Unit, group.Key.Region, start, (ushort)block.End); continue; } catch (Exception ex) @@ -1292,12 +1292,21 @@ public sealed class ModbusDriver batch.Answered = true; foreach (var (idx, tag) in block.Members) { - var sliceOffsetBytes = (tag.Address - block.Start) * 2; - var sliceLenBytes = RegisterCount(tag) * 2; handled.Add(idx); try { - var value = Decode(() => DecodeRegister(data.AsSpan(sliceOffsetBytes, sliceLenBytes), tag), tag); + object value; + if (isBits) + { + var bit = tag.Address - block.Start; + value = ((data[bit / 8] >> (bit % 8)) & 0x01) == 1; + } + else + { + var sliceOffsetBytes = (tag.Address - block.Start) * 2; + var sliceLenBytes = RegisterCount(tag) * 2; + value = Decode(() => DecodeRegister(data.AsSpan(sliceOffsetBytes, sliceLenBytes), tag), tag); + } results[idx] = new DataValueSnapshot(value, 0u, timestamp, timestamp); InvalidateWriteCacheIfDiverged(fullReferences[idx], value); } @@ -1313,6 +1322,24 @@ public sealed class ModbusDriver return handled; } + private static bool IsBitRegion(ModbusRegion region) => region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs; + + private static byte ReadFunctionCode(ModbusRegion region) => region switch + { + ModbusRegion.Coils => 0x01, + ModbusRegion.DiscreteInputs => 0x02, + ModbusRegion.HoldingRegisters => 0x03, + _ => 0x04, + }; + + /// Reads one contiguous range of the region with the matching function code — registers as + /// big-endian bytes, bits as an LSB-first bitmap. + private Task ReadRangeAsync( + IModbusTransport transport, byte unit, ModbusRegion region, ushort start, ushort qty, CancellationToken ct) + => IsBitRegion(region) + ? ReadBitBlockAsync(transport, unit, ReadFunctionCode(region), start, qty, ct) + : ReadRegisterBlockAsync(transport, unit, ReadFunctionCode(region), start, qty, ct); + private void InvalidateWriteCacheIfDiverged(string fullRef, object value) { if (!_options.WriteOnChangeOnly) return; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs index 31e33d69..9aa8e0d7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs @@ -86,7 +86,8 @@ public static class ModbusDriverFactoryExtensions UseFC16ForSingleRegisterWrites = dto.UseFC16ForSingleRegisterWrites ?? false, DisableFC23 = dto.DisableFC23 ?? false, WriteOnChangeOnly = dto.WriteOnChangeOnly ?? false, - MaxReadGap = dto.MaxReadGap ?? 0, + MaxReadGap = dto.MaxReadGap ?? ModbusDriverOptions.DefaultMaxReadGap, + DisableReadCoalescing = dto.DisableReadCoalescing ?? false, Family = dto.Family is null ? ModbusFamily.Generic : ParseEnum(dto.Family, "", driverInstanceId, "Family"), MelsecSubFamily = dto.MelsecSubFamily is null ? MelsecFamily.Q_L_iQR @@ -171,6 +172,8 @@ public static class ModbusDriverFactoryExtensions public bool? WriteOnChangeOnly { get; init; } /// Gets or sets the maximum gap between register addresses for coalescing. public ushort? MaxReadGap { get; init; } + /// Gets or sets a value indicating whether block-read coalescing is turned off entirely. + public bool? DisableReadCoalescing { get; init; } /// Gets or sets the Modbus family type. public string? Family { get; init; } /// Gets or sets the Melsec subfamily. diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor index 0c34a768..a46a255e 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor @@ -80,7 +80,13 @@
-
0 = no coalescing. Typical 5–32.
+
Default 8. 0 = adjacent tags only. Refused ranges fall back per tag.
+
+
+
+ + +
@@ -311,7 +317,8 @@ public int MaxRegistersPerRead { get; set; } = 125; public int MaxRegistersPerWrite { get; set; } = 123; public int MaxCoilsPerRead { get; set; } = 2000; - public int MaxReadGap { get; set; } = 0; + public int MaxReadGap { get; set; } = ModbusDriverOptions.DefaultMaxReadGap; + public bool DisableReadCoalescing { get; set; } = false; // Write options public bool UseFC15ForSingleCoilWrites { get; set; } = false; @@ -357,6 +364,7 @@ MaxRegistersPerWrite = o.MaxRegistersPerWrite, MaxCoilsPerRead = o.MaxCoilsPerRead, MaxReadGap = o.MaxReadGap, + DisableReadCoalescing = o.DisableReadCoalescing, UseFC15ForSingleCoilWrites = o.UseFC15ForSingleCoilWrites, UseFC16ForSingleRegisterWrites = o.UseFC16ForSingleRegisterWrites, DisableFC23 = o.DisableFC23, @@ -399,6 +407,7 @@ AutoProhibitReprobeInterval = AutoProhibitReprobeIntervalSeconds > 0 ? TimeSpan.FromSeconds(AutoProhibitReprobeIntervalSeconds) : null, MaxReadGap = (ushort)Math.Clamp(MaxReadGap, 0, 65535), + DisableReadCoalescing = DisableReadCoalescing, Family = Family, MelsecSubFamily = MelsecSubFamily, Transport = Transport, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs new file mode 100644 index 00000000..45940c37 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs @@ -0,0 +1,155 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Review 07 U2: block-read coalescing is ON by default (gap 8), and it covers coils, discrete inputs, +/// BitInRegister and String tags — 16 bit tags on one status word used to cost 16 reads of +/// the same register, and 64 adjacent coils 64 FC01s. +/// +[Trait("Category", "Unit")] +public sealed class ModbusCoalescingDefaultsTests +{ + private static async Task<(ModbusDriver, ScriptedTransport)> StartAsync(ModbusDriverOptions opts) + { + var t = new ScriptedTransport(); + var d = new ModbusDriver(opts, "m1", _ => t); + await d.InitializeAsync("{}", TestContext.Current.CancellationToken); + return (d, t); + } + + private static ModbusDriverOptions Opts(IEnumerable tags, bool disable = false) => new() + { + Host = "f", RawTags = ModbusRawTags.Entries(tags), DisableReadCoalescing = disable, + Probe = new ModbusProbeOptions { Enabled = false }, + }; + + [Fact] + public void Default_options_coalesce_with_a_gap_of_8() + { + new ModbusDriverOptions().MaxReadGap.ShouldBe((ushort)8); + new ModbusDriverOptions().DisableReadCoalescing.ShouldBeFalse(); + ModbusDriverFactoryExtensions.ParseOptions("m1", """{"host":"h"}""").MaxReadGap.ShouldBe((ushort)8); + } + + [Fact] + public async Task Default_configured_holding_registers_are_read_in_one_PDU() + { + var tags = Enumerable.Range(0, 10).Select(i => + new ModbusTagDefinition($"HR{i}", ModbusRegion.HoldingRegisters, (ushort)(100 + i * 3), ModbusDataType.Int16)).ToArray(); + var (drv, t) = await StartAsync(Opts(tags)); + await using var _ = drv; + for (var i = 0; i < 10; i++) t.Bank.HoldingRegisters[100 + i * 3] = (ushort)(i + 1); + + var r = await drv.ReadAsync([.. tags.Select(x => x.Name)], TestContext.Current.CancellationToken); + + t.Requests(0x03).Count.ShouldBe(1); + r.Select(s => (short)s.Value!).ShouldBe(Enumerable.Range(1, 10).Select(i => (short)i)); + } + + [Fact] + public async Task DisableReadCoalescing_reads_every_tag_alone() + { + // Positive control for the default-on test: the off switch really is per-tag. + var tags = new[] + { + new ModbusTagDefinition("A", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16), + new ModbusTagDefinition("B", ModbusRegion.HoldingRegisters, 101, ModbusDataType.Int16), + }; + var (drv, t) = await StartAsync(Opts(tags, disable: true)); + await using var _ = drv; + await drv.ReadAsync(["A", "B"], TestContext.Current.CancellationToken); + t.Requests(0x03).Count.ShouldBe(2); + } + + [Theory] + [InlineData(ModbusRegion.Coils, (byte)0x01)] + [InlineData(ModbusRegion.DiscreteInputs, (byte)0x02)] + public async Task Adjacent_bit_tags_fold_into_one_bit_read(ModbusRegion region, byte fc) + { + var tags = Enumerable.Range(0, 64).Select(i => + new ModbusTagDefinition($"B{i}", region, (ushort)(10 + i), ModbusDataType.Bool)).ToArray(); + var (drv, t) = await StartAsync(Opts(tags)); + await using var _ = drv; + var bank = region == ModbusRegion.Coils ? t.Bank.Coils : t.Bank.DiscreteInputs; + for (var i = 0; i < 64; i++) bank[10 + i] = i % 3 == 0; + + var r = await drv.ReadAsync([.. tags.Select(x => x.Name)], TestContext.Current.CancellationToken); + + t.Requests(fc).ShouldBe([(fc, (ushort)10, (ushort)64)]); + for (var i = 0; i < 64; i++) r[i].Value.ShouldBe(i % 3 == 0, $"bit {i}"); + } + + [Fact] + public async Task Bit_tags_on_one_status_word_fold_into_one_register_read() + { + var tags = Enumerable.Range(0, 16).Select(b => + new ModbusTagDefinition($"S{b}", ModbusRegion.HoldingRegisters, 40, ModbusDataType.BitInRegister, BitIndex: (byte)b)).ToArray(); + var (drv, t) = await StartAsync(Opts(tags)); + await using var _ = drv; + t.Bank.HoldingRegisters[40] = 0b1010_0000_0000_0101; + + var r = await drv.ReadAsync([.. tags.Select(x => x.Name)], TestContext.Current.CancellationToken); + + t.Requests(0x03).ShouldBe([((byte)0x03, (ushort)40, (ushort)1)]); + for (var b = 0; b < 16; b++) r[b].Value.ShouldBe(((0b1010_0000_0000_0101 >> b) & 1) == 1, $"bit {b}"); + } + + [Fact] + public async Task A_string_tag_coalesces_with_its_neighbours() + { + var tags = new[] + { + new ModbusTagDefinition("Num", ModbusRegion.HoldingRegisters, 50, ModbusDataType.Int16), + new ModbusTagDefinition("Str", ModbusRegion.HoldingRegisters, 51, ModbusDataType.String, StringLength: 4), + new ModbusTagDefinition("After", ModbusRegion.HoldingRegisters, 53, ModbusDataType.UInt16), + }; + var (drv, t) = await StartAsync(Opts(tags)); + await using var _ = drv; + t.Bank.HoldingRegisters[50] = 7; + t.Bank.HoldingRegisters[51] = ('A' << 8) | 'B'; + t.Bank.HoldingRegisters[52] = ('C' << 8) | 'D'; + t.Bank.HoldingRegisters[53] = 9; + + var r = await drv.ReadAsync(["Num", "Str", "After"], TestContext.Current.CancellationToken); + + t.Requests(0x03).ShouldBe([((byte)0x03, (ushort)50, (ushort)4)]); + r[0].Value.ShouldBe((short)7); + r[1].Value.ShouldBe("ABCD"); + r[2].Value.ShouldBe((ushort)9); + } + + [Fact] + public async Task A_refused_coil_range_is_prohibited_and_reprobed_with_FC01() + { + var tags = Enumerable.Range(0, 4).Select(i => + new ModbusTagDefinition($"C{i}", ModbusRegion.Coils, (ushort)(20 + i), ModbusDataType.Bool)).ToArray(); + var opts = new ModbusDriverOptions + { + Host = "f", RawTags = ModbusRawTags.Entries(tags), Probe = new ModbusProbeOptions { Enabled = false }, + AutoProhibitReprobeInterval = TimeSpan.FromHours(1), + }; + var (drv, t) = await StartAsync(opts); + await using var _ = drv; + var refuseMerged = true; + t.Handler = (_, pdu, _) => + { + var qty = (pdu[3] << 8) | pdu[4]; + return pdu[0] == 0x01 && qty > 1 && refuseMerged + ? Task.FromException(new ModbusException(0x01, 0x02, "protected")) + : null; + }; + + var r = await drv.ReadAsync([.. tags.Select(x => x.Name)], TestContext.Current.CancellationToken); + r.ShouldAllBe(s => s.StatusCode == 0u, "the per-tag fallback still reads every coil"); + drv.AutoProhibitedRangeCount.ShouldBe(1); + + refuseMerged = false; + while (t.Sent.TryDequeue(out var _1)) { } + await drv.RunReprobeOnceForTestAsync(TestContext.Current.CancellationToken); + t.Sent.ShouldAllBe(s => s.Pdu[0] == 0x01, "a coil range is re-probed with FC01, never a register read"); + drv.AutoProhibitedRangeCount.ShouldBe(0); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs index 8bdea55f..72542014 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs @@ -42,9 +42,9 @@ public sealed class ModbusCoalescingTests public ValueTask DisposeAsync() => ValueTask.CompletedTask; } - /// Verifies that MaxReadGap=0 defaults to per-tag reads without coalescing. + /// MaxReadGap=0 coalesces only adjacent tags: a one-register hole splits the read. [Fact] - public async Task MaxReadGap_Zero_Defaults_To_Per_Tag_Reads() + public async Task MaxReadGap_Zero_Does_Not_Bridge_A_Gap() { var fake = new CountingTransport(); var t1 = new ModbusTagDefinition("T1", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16); @@ -56,7 +56,7 @@ public sealed class ModbusCoalescingTests await drv.ReadAsync(["T1", "T2"], CancellationToken.None); - // With coalescing off, expect 2 separate FC03 reads. + // A one-register gap with MaxReadGap=0: expect 2 separate FC03 reads. var fc03Reads = fake.Reads.Where(r => r.Fc == 0x03).ToList(); fc03Reads.Count.ShouldBe(2); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs index e865bc34..b05a5ffd 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ScriptedTransport.cs @@ -30,6 +30,13 @@ internal sealed class ScriptedTransport : IModbusTransport public int TotalSends => _sendsByUnit.Values.Sum(); + /// Every PDU sent, in order. + public ConcurrentQueue<(byte Unit, byte[] Pdu)> Sent { get; } = new(); + + /// The (function code, start, quantity) of every sent request with that function code. + public List<(byte Fc, ushort Start, ushort Qty)> Requests(byte fc) => + [.. Sent.Where(s => s.Pdu[0] == fc).Select(s => (s.Pdu[0], (ushort)((s.Pdu[1] << 8) | s.Pdu[2]), (ushort)((s.Pdu[3] << 8) | s.Pdu[4])))]; + public async Task ConnectAsync(CancellationToken ct) { ConnectEntered.TrySetResult(); @@ -39,6 +46,7 @@ internal sealed class ScriptedTransport : IModbusTransport public Task SendAsync(byte unitId, byte[] pdu, CancellationToken ct) { _sendsByUnit.AddOrUpdate(unitId, 1, (_, n) => n + 1); + Sent.Enqueue((unitId, pdu)); var scripted = Handler?.Invoke(unitId, pdu, ct); return scripted ?? Bank.SendAsync(unitId, pdu, ct); } From 43f45bb9781e1aaad731f1d2ba61d90ea898becc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:26:55 -0400 Subject: [PATCH 07/34] fix(mqtt): dispose an unpublished connection and stamp lifecycle generations (R06:M-3, M-10) A refused initial connect (or a failed Sparkplug establish) now disposes the MqttConnection it built. Every lifecycle operation takes a generation (initialize inside the gate; shutdown/dispose before waiting for it): a wedged initialize that completes after a timed-out ShutdownAsync's ungated teardown releases its session instead of publishing it, and a superseded operation's health/host-state writes are refused. TeardownAsync captures every field before disposing so an ungated teardown never reaches into a later initialize's state. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../MqttDriver.cs | 139 +++++++++++++++--- .../MqttDriverSessionLifecycleTests.cs | 69 +++++++++ 2 files changed, 186 insertions(+), 22 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs index b8e92621..26cdeab6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs @@ -208,6 +208,15 @@ public sealed class MqttDriver private SparkplugPrimaryHost? _primaryHost; private DriverHealth _health = new(DriverState.Unknown, null, null); + + /// Guards the generation bump against the generation-checked health write. + private readonly object _healthLock = new(); + + /// + /// Bumped by every lifecycle operation (initialize inside the gate; shutdown and dispose before + /// waiting for it). The stamp that keeps a superseded operation from publishing state. + /// + private long _lifecycleGeneration; private HostState _hostState = HostState.Unknown; private DateTime _hostStateChangedUtc = DateTime.UtcNow; private int _disposed; @@ -289,6 +298,9 @@ public sealed class MqttDriver /// internal Func? BeforeConnectHookForTests { get; set; } + /// Test seam: observes every an initialize builds. Null in production. + internal Action? ConnectionCreatedForTests { get; set; } + // ---- IDriver: lifecycle ---- /// @@ -387,6 +399,11 @@ public sealed class MqttDriver // watchdog's reconnect) is measured against, reporting a driver that was delivering a moment ago // as one that has never delivered. var lastMessage = GetHealth().LastSuccessfulRead; + + // Taken BEFORE the gate wait: a wedged initialize holding the gate is superseded by this call, so + // when it finally completes it disposes the session it built instead of publishing it behind the + // ungated teardown below (R06:M-10), and its late health write is refused. + var generation = BeginLifecycleOperation(); if (await _lifecycleGate.WaitAsync(TimeSpan.FromSeconds(_options.ConnectTimeoutSeconds), cancellationToken) .ConfigureAwait(false)) { @@ -408,8 +425,10 @@ public sealed class MqttDriver await TeardownAsync().ConfigureAwait(false); } - WriteHealth(new DriverHealth(DriverState.Unknown, lastMessage, null)); - TransitionHost(HostState.Unknown); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, lastMessage, null))) + { + TransitionHost(HostState.Unknown); + } } /// @@ -911,6 +930,9 @@ public sealed class MqttDriver return; } + // Supersedes any in-flight lifecycle operation, exactly as ShutdownAsync does. + BeginLifecycleOperation(); + if (await _lifecycleGate.WaitAsync(TimeSpan.FromSeconds(_options.ConnectTimeoutSeconds)) .ConfigureAwait(false)) { @@ -940,7 +962,12 @@ public sealed class MqttDriver /// The gate-held body of . private async Task InitializeCoreAsync(string driverConfigJson, CancellationToken cancellationToken) { - WriteHealth(new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); + // Taken INSIDE the gate (the caller holds it), so a second initialize queued behind this one does + // not supersede it; only a ShutdownAsync / DisposeAsync — which take theirs before waiting — can. + var generation = BeginLifecycleOperation(); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); + MqttConnection? built = null; + var published = false; try { // Tear down whatever this driver already holds BEFORE building a new session. Both fields @@ -965,6 +992,8 @@ public sealed class MqttDriver RegisterAuthoredTags(_options); var connection = new MqttConnection(_options, _driverInstanceId, _logger); + built = connection; + ConnectionCreatedForTests?.Invoke(connection); // MUST precede the connect: this wires message delivery AND the reconnect re-subscribe. // The handler is passed through unwrapped on purpose — its throw-on-total-failure is what @@ -992,10 +1021,13 @@ public sealed class MqttDriver // disposing what we just built, never leaking it. if (Volatile.Read(ref _disposed) != 0) { - await connection.DisposeAsync().ConfigureAwait(false); throw new ObjectDisposedException(nameof(MqttDriver)); } + // The same race against a ShutdownAsync whose bounded gate wait expired: it has already torn + // down ungated, so publishing now would leave a live session no later shutdown can see. + ThrowIfSuperseded(generation); + // Sparkplug subscribes ONCE, here, rather than on the OPC UA server's first subscribe: the // driver must see birth certificates whether or not anything is subscribed yet, because a // birth is where every alias and datatype comes from. This also issues the late-join @@ -1008,28 +1040,44 @@ public sealed class MqttDriver // MqttConnection's own remarks describe having already fixed once. if (_sparkplug is { } sparkplug) { - try - { - await sparkplug.EstablishAsync(cancellationToken).ConfigureAwait(false); - } - catch - { - await connection.DisposeAsync().ConfigureAwait(false); - throw; - } + await sparkplug.EstablishAsync(cancellationToken).ConfigureAwait(false); } + ThrowIfSuperseded(generation); _connection = connection; + published = true; + + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Healthy, connection.LastMessageUtc, null))) + { + TransitionHost(HostState.Running); + } - WriteHealth(new DriverHealth(DriverState.Healthy, connection.LastMessageUtc, null)); - TransitionHost(HostState.Running); StartHostProbe(); await StartPrimaryHostAsync(cancellationToken).ConfigureAwait(false); + + // Re-checked after publishing: a shutdown whose gate wait expired between the check above and + // here captured whatever it found and left these fields for us. We still hold the gate, so no + // newer initialize can have published anything — everything in them is ours to release. + if (!IsCurrentLifecycle(generation)) + { + await TeardownAsync().ConfigureAwait(false); + ThrowIfSuperseded(generation); + } } catch (Exception ex) { - WriteHealth(new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, ex.Message)); - TransitionHost(HostState.Faulted); + // R06:M-3: a connection built but never published is disposed here — a refused connect, a + // failed Sparkplug establish, a supersede. The published one belongs to TeardownAsync. + if (!published && built is not null) + { + await built.DisposeAsync().ConfigureAwait(false); + } + + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, ex.Message))) + { + TransitionHost(HostState.Faulted); + } + throw; } } @@ -1380,24 +1428,28 @@ public sealed class MqttDriver /// Shared teardown for , and the session rebuild. private async Task TeardownAsync() { + // Every field is CAPTURED before anything is disposed. A shutdown that timed out on the gate runs + // this ungated; capturing up front means it only ever releases what existed when it started, and + // never reaches into fields a later initialize publishes while the slow disposals below run. + var primaryHost = Interlocked.Exchange(ref _primaryHost, null); + var probe = Interlocked.Exchange(ref _hostProbeCts, null); + var connection = Interlocked.Exchange(ref _connection, null); + // Stand the host claim down FIRST, while the broker is still reachable and before the ingest // session goes. Its explicit retained OFFLINE is what makes the transition immediate for edge // nodes; leaving it to the Will costs them up to 1.5 keep-alive periods of believing a host // that has already stopped is still online. - var primaryHost = Interlocked.Exchange(ref _primaryHost, null); if (primaryHost is not null) { await primaryHost.DisposeAsync().ConfigureAwait(false); } - var probe = Interlocked.Exchange(ref _hostProbeCts, null); if (probe is not null) { await probe.CancelAsync().ConfigureAwait(false); probe.Dispose(); } - var connection = Interlocked.Exchange(ref _connection, null); if (connection is not null) { await connection.DisposeAsync().ConfigureAwait(false); @@ -1479,6 +1531,49 @@ public sealed class MqttDriver /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); - /// Barrier-protected publish of a new health snapshot. - private void WriteHealth(DriverHealth value) => Volatile.Write(ref _health, value); + /// + /// Starts a lifecycle operation and returns its generation. Every later operation supersedes it: + /// its health / host-state writes are refused and an initialize stops before publishing a session. + /// + private long BeginLifecycleOperation() + { + lock (_healthLock) + { + return ++_lifecycleGeneration; + } + } + + /// Whether is still the newest lifecycle operation. + private bool IsCurrentLifecycle(long generation) => Interlocked.Read(ref _lifecycleGeneration) == generation; + + /// Throws when a newer lifecycle operation has superseded . + private void ThrowIfSuperseded(long generation) + { + if (!IsCurrentLifecycle(generation)) + { + throw new OperationCanceledException( + $"MQTT driver '{_driverInstanceId}': this initialize was superseded by a later shutdown/dispose; " + + "the session it built has been released."); + } + } + + /// + /// Publishes a health snapshot only while is the newest lifecycle + /// operation — the check and the write are atomic against , so a + /// superseded initialize or shutdown can never overwrite a newer one's state. + /// + /// when the write was applied. + private bool WriteHealthIfCurrent(long generation, DriverHealth value) + { + lock (_healthLock) + { + if (_lifecycleGeneration != generation) + { + return false; + } + + Volatile.Write(ref _health, value); + return true; + } + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 947fb081..76d12cdd 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -219,6 +219,75 @@ public sealed class MqttDriverSessionLifecycleTests } } + /// + /// R06:M-3. A refused initial connect used to rethrow without disposing the + /// (and its IMqttClient) it had just built. Control: a + /// successful initialize leaves its connection live, so the seam is not reporting "disposed" for + /// everything. + /// + [Fact] + public async Task InitializeAsync_RefusedConnect_DisposesTheConnectionItBuilt() + { + using var broker = new MiniBroker { ConnAckReturnCode = MiniBroker.ReturnCodeNotAuthorized }; + var options = BrokerOptions(broker.Port); + await using var driver = new MqttDriver(options, "d", null); + var built = new List(); + driver.ConnectionCreatedForTests = c => built.Add(c); + + await Should.ThrowAsync(() => driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken)); + + built.ShouldHaveSingleItem().State.ShouldBe(MqttConnectionState.Disposed); + + broker.ConnAckReturnCode = MiniBroker.ReturnCodeAccepted; + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + built.Count.ShouldBe(2); + built[1].State.ShouldBe(MqttConnectionState.Connected); + } + + /// + /// R06:M-10 + the lifecycle-generation rule. A shutdown that times out on the lifecycle gate + /// tears down ungated; the wedged initialize that held the gate must then NOT publish its freshly + /// connected session behind that teardown, and its late health write must not overwrite the + /// shutdown's. The positive controls: the wedged init really did reach the broker (AcceptedCount), + /// and a later initialize connects and reports Healthy. + /// + [Trait("Category", "SlowUnit")] + [Fact] + public async Task AWedgedInit_CompletingAfterATimedOutShutdown_PublishesNothing() + { + using var broker = new MiniBroker(); + var options = BrokerOptions(broker.Port) with { ConnectTimeoutSeconds = 1 }; + await using var driver = new MqttDriver(options, "d", null); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + driver.BeforeConnectHookForTests = async _ => + { + entered.TrySetResult(); + await release.Task; + }; + + var initialize = Task.Run(() => driver.InitializeAsync(Json(options), CancellationToken.None)); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // Times out on the gate (1 s) and tears down ungated. + await driver.ShutdownAsync(TestContext.Current.CancellationToken); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + + driver.BeforeConnectHookForTests = null; + release.SetResult(); + await Should.ThrowAsync(() => initialize); + + broker.AcceptedCount.ShouldBe(1, "the wedged initialize did connect — the control for the absence below"); + await WaitForAsync(() => broker.LiveConnections == 0, "the superseded session to be torn down"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown, "a superseded init must not overwrite newer health"); + driver.GetHostStatuses().ShouldHaveSingleItem().State.ShouldBe(HostState.Unknown); + + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + await WaitForAsync(() => broker.LiveConnections == 1, "the fresh session"); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); From 1adc9d65e1f6dfb9560e35861ae35b680241a0d4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:26:59 -0400 Subject: [PATCH 08/34] feat(modbus): String[] and BitInRegister[] writes U3: both array shapes read fine but every write failed BadInternalError. String[] now encodes each element at its register width (StringLength chars, 2 per register); BitInRegister[] reads the span once, flips the addressed bit of each register and writes the span back with FC16 (per-register RMW locks taken in ascending order). A length mismatch is BadOutOfRange and sends nothing. DriverArrayWriteSupport (outside this lane) still advertises these nodes read-only - recorded as follow-up F1 in the parity test. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriver.cs | 65 ++++++++++++++- .../ModbusArrayWriteSupportParityTests.cs | 4 +- .../ModbusStringAndBitArrayWriteTests.cs | 82 +++++++++++++++++++ 3 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusStringAndBitArrayWriteTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index cc252539..4aa312d3 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -1554,7 +1554,10 @@ public sealed class ModbusDriver if (tag.DataType == ModbusDataType.BitInRegister && tag.Region is ModbusRegion.HoldingRegisters) { - await WriteBitInRegisterAsync(transport, tag, value, ct).ConfigureAwait(false); + if (tag.ArrayCount.HasValue) + await WriteBitInRegisterArrayAsync(transport, tag, value, ct).ConfigureAwait(false); + else + await WriteBitInRegisterAsync(transport, tag, value, ct).ConfigureAwait(false); return; } @@ -1680,9 +1683,67 @@ public sealed class ModbusDriver ModbusDataType.Int16 or ModbusDataType.UInt16 or ModbusDataType.Bcd16 => 2, ModbusDataType.Int32 or ModbusDataType.UInt32 or ModbusDataType.Float32 or ModbusDataType.Bcd32 => 4, ModbusDataType.Int64 or ModbusDataType.UInt64 or ModbusDataType.Float64 => 8, - _ => throw new InvalidOperationException($"Element byte count not defined for {tag.DataType} in array context"), + // A String element is StringLength chars at 2 per register, rounded up — the read path's width. + ModbusDataType.String => RegisterCount(tag) * 2, + _ => throw new ModbusWriteRefusedException(StatusBadNotSupported, + $"Array writes are not supported for {tag.DataType} tags ({tag.Name})"), }; + /// + /// BitInRegister[] write: element i is bit BitIndex of holding register + /// Address + i (the read path's layout). One FC03 of the whole span, flip the addressed bit of + /// each register, one FC16 back — so the other 15 bits of every register are preserved. Serialised + /// per register with the scalar bit-write path (locks taken in ascending address order, so two + /// overlapping array writes cannot deadlock). Not atomic against the PLC's own writes between the + /// read and the write, exactly like the scalar RMW. + /// + private async Task WriteBitInRegisterArrayAsync( + IModbusTransport transport, ModbusTagDefinition tag, object? value, CancellationToken ct) + { + var count = tag.ArrayCount!.Value; + var bits = ToBoolArray(value, count, tag.Name); + var writeCap = _options.MaxRegistersPerWrite == 0 ? (ushort)123 : _options.MaxRegistersPerWrite; + if (count > writeCap || tag.Address + count - 1 > ushort.MaxValue) + throw new ModbusWriteRefusedException(StatusBadOutOfRange, + $"BitInRegister array write of {count} registers to {tag.Name} exceeds MaxRegistersPerWrite={writeCap}"); + if (tag.BitIndex > 15) + throw new ModbusWriteRefusedException(StatusBadOutOfRange, + $"BitInRegister bit index {tag.BitIndex} out of range (0-15) for tag {tag.Name}."); + + var locks = new List(count); + try + { + for (var i = 0; i < count; i++) + { + var l = GetRmwLock((ushort)(tag.Address + i)); + await l.WaitAsync(ct).ConfigureAwait(false); + locks.Add(l); + } + + var unit = ResolveUnitId(tag); + var current = await ReadRegisterBlockAsync(transport, unit, 0x03, tag.Address, (ushort)count, ct).ConfigureAwait(false); + var payload = new byte[count * 2]; + for (var i = 0; i < count; i++) + { + var reg = (ushort)((current[i * 2] << 8) | current[i * 2 + 1]); + reg = bits[i] ? (ushort)(reg | (1 << tag.BitIndex)) : (ushort)(reg & ~(1 << tag.BitIndex)); + payload[i * 2] = (byte)(reg >> 8); + payload[i * 2 + 1] = (byte)(reg & 0xFF); + } + var pdu = new byte[6 + 1 + payload.Length]; + pdu[0] = 0x10; + pdu[1] = (byte)(tag.Address >> 8); pdu[2] = (byte)(tag.Address & 0xFF); + pdu[3] = (byte)(count >> 8); pdu[4] = (byte)(count & 0xFF); + pdu[5] = (byte)payload.Length; + Buffer.BlockCopy(payload, 0, pdu, 6, payload.Length); + await transport.SendAsync(unit, pdu, ct).ConfigureAwait(false); + } + finally + { + for (var i = locks.Count - 1; i >= 0; i--) locks[i].Release(); + } + } + /// /// Read-modify-write one bit in a holding register. FC03 → bit-swap → FC06. Serialised /// against other bit writes targeting the same register via . diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs index 6d2363b1..41a20e50 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusArrayWriteSupportParityTests.cs @@ -35,10 +35,8 @@ public sealed class ModbusArrayWriteSupportParityTests /// private static bool PendingF1(ModbusRegion region, ModbusDataType dataType) => (region == ModbusRegion.Coils && dataType != ModbusDataType.Bool) - || (region == ModbusRegion.HoldingRegisters && dataType is ModbusDataType.String or ModbusDataType.BitInRegister && PendingU3); + || (region == ModbusRegion.HoldingRegisters && dataType is ModbusDataType.String or ModbusDataType.BitInRegister); - // Flipped to true by the U3 commit (String[] / BitInRegister[] encoders). - private const bool PendingU3 = false; public static TheoryData PendingShapes() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusStringAndBitArrayWriteTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusStringAndBitArrayWriteTests.cs new file mode 100644 index 00000000..8cdfa7d6 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusStringAndBitArrayWriteTests.cs @@ -0,0 +1,82 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Review 07 U3: String[] and BitInRegister[] tags read fine; their writes used to fail +/// BadInternalError (no element width for String, Convert.ToBoolean(bool[]) for the bit +/// array). Both now encode count elements at the tag's element width; a length mismatch is +/// BadOutOfRange. +/// +[Trait("Category", "Unit")] +public sealed class ModbusStringAndBitArrayWriteTests +{ + private static async Task<(ModbusDriver, ScriptedTransport)> StartAsync(params ModbusTagDefinition[] tags) + { + var t = new ScriptedTransport(); + var d = new ModbusDriver(new ModbusDriverOptions + { + Host = "f", RawTags = ModbusRawTags.Entries(tags), Probe = new ModbusProbeOptions { Enabled = false }, + }, "m1", _ => t); + await d.InitializeAsync("{}", TestContext.Current.CancellationToken); + return (d, t); + } + + [Fact] + public async Task String_array_write_encodes_each_element_at_its_register_width() + { + var tag = new ModbusTagDefinition("S", ModbusRegion.HoldingRegisters, 10, ModbusDataType.String, + StringLength: 3, ArrayCount: 2); // 2 registers per element + var (drv, t) = await StartAsync(tag); + await using var _ = drv; + + var w = await drv.WriteAsync([new WriteRequest("S", new[] { "AB", "XYZ" })], TestContext.Current.CancellationToken); + + w[0].StatusCode.ShouldBe(0u); + t.Bank.HoldingRegisters[10].ShouldBe((ushort)(('A' << 8) | 'B')); + t.Bank.HoldingRegisters[11].ShouldBe((ushort)0); + t.Bank.HoldingRegisters[12].ShouldBe((ushort)(('X' << 8) | 'Y')); + t.Bank.HoldingRegisters[13].ShouldBe((ushort)('Z' << 8)); + // Round trip through the read path. + (await drv.ReadAsync(["S"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(new[] { "AB", "XYZ" }); + } + + [Fact] + public async Task BitInRegister_array_write_sets_only_the_addressed_bit_of_each_register() + { + var tag = new ModbusTagDefinition("B", ModbusRegion.HoldingRegisters, 30, ModbusDataType.BitInRegister, + BitIndex: 4, ArrayCount: 3); + var (drv, t) = await StartAsync(tag); + await using var _ = drv; + t.Bank.HoldingRegisters[30] = 0xFFFF; + t.Bank.HoldingRegisters[31] = 0x0001; + t.Bank.HoldingRegisters[32] = 0x8000; + + var w = await drv.WriteAsync([new WriteRequest("B", new object[] { false, true, 1 })], TestContext.Current.CancellationToken); + + w[0].StatusCode.ShouldBe(0u); + t.Bank.HoldingRegisters[30].ShouldBe((ushort)0xFFEF, "bit 4 cleared, neighbours kept"); + t.Bank.HoldingRegisters[31].ShouldBe((ushort)0x0011); + t.Bank.HoldingRegisters[32].ShouldBe((ushort)0x8010); + (await drv.ReadAsync(["B"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(new[] { false, true, true }); + } + + [Theory] + [InlineData("S")] + [InlineData("B")] + public async Task A_length_mismatch_is_BadOutOfRange_and_sends_nothing(string tagName) + { + var (drv, t) = await StartAsync( + new ModbusTagDefinition("S", ModbusRegion.HoldingRegisters, 10, ModbusDataType.String, StringLength: 2, ArrayCount: 2), + new ModbusTagDefinition("B", ModbusRegion.HoldingRegisters, 30, ModbusDataType.BitInRegister, BitIndex: 1, ArrayCount: 3)); + await using var _ = drv; + object value = tagName == "S" ? new[] { "only one" } : new[] { true }; + + var w = await drv.WriteAsync([new WriteRequest(tagName, value)], TestContext.Current.CancellationToken); + + w[0].StatusCode.ShouldBe(0x803C0000u); + t.Sent.Where(s => s.Pdu[0] is 0x06 or 0x10).ShouldBeEmpty(); + } +} From 57ffbf79e5609738337a77b4906483445c1796c1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:28:33 -0400 Subject: [PATCH 09/34] fix(mqtt): prune LastValueCache to the authored set on Register (R06:M-2) Both ingest paths now drop cache entries whose RawPath is no longer authored when a deploy re-registers, so a removed tag's value and every non-authored subscribe reference's BadNodeIdUnknown marker no longer accumulate for the driver's lifetime. Still-authored tags keep their observed values. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../LastValueCache.cs | 24 +++++++++++++++++ .../MqttSubscriptionManager.cs | 5 ++++ .../Sparkplug/SparkplugIngestor.cs | 5 ++++ .../MqttSubscriptionManagerTests.cs | 27 +++++++++++++++++++ .../SparkplugIngestorTests.cs | 22 +++++++++++++++ 5 files changed, 83 insertions(+) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs index 7df9dd0e..5ba310bf 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs @@ -45,6 +45,30 @@ public sealed class LastValueCache _values[rawPath] = snapshot; } + /// How many RawPaths currently hold an entry. Diagnostics and the bounded-growth tests. + internal int Count => _values.Count; + + /// + /// Drops every entry whose RawPath rejects — called by each ingest path's + /// Register with "is this still an authored tag", so the cache is bounded by the CURRENT + /// authored set rather than by the union of every RawPath (and every non-authored reference ever + /// subscribed) across the driver's lifetime (R06:M-2). + /// + /// Returns for a RawPath whose entry must survive. + /// How many entries were dropped. + public int RetainOnly(Func keep) + { + ArgumentNullException.ThrowIfNull(keep); + var dropped = 0; + foreach (var rawPath in _values.Keys) + { + if (!keep(rawPath) && _values.TryRemove(rawPath, out _)) + dropped++; + } + + return dropped; + } + /// /// Read the last observed value for . Never throws: a /// null/empty or a RawPath never observed both return a diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 9e4d8f64..1504328a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -325,6 +325,11 @@ public sealed class MqttSubscriptionManager _authored = table; _warnedRefs.Clear(); + // Bound the shared last-value cache by the CURRENT authored set (R06:M-2): a dropped tag's value + // and every non-authored reference's BadNodeIdUnknown marker would otherwise live forever. A + // still-authored tag keeps its value, so a redeploy never blanks what has been observed. + Values.RetainOnly(table.ByRawPath.ContainsKey); + // Drop state belonging to tags this deploy removed. Without this, a topic whose last tag was // deleted stays in _subscribedTopics and is dutifully re-SUBSCRIBEd on every future reconnect, // forever — and its RawPaths keep a handle mapping that can never fire again. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs index cd368eae..5d63e1b7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs @@ -416,6 +416,11 @@ public sealed class SparkplugIngestor _authored = table; _warned.Clear(); + // Bound the shared last-value cache by the CURRENT authored set (R06:M-2): a dropped tag's value + // and every non-authored reference's BadNodeIdUnknown marker would otherwise live forever. A + // still-authored tag keeps its value, so a redeploy never blanks what has been observed. + Values.RetainOnly(table.ByRawPath.ContainsKey); + // Drop state belonging to edge nodes this deploy removed. Without this a tracker (and its // rebirth debounce slot) for a node nobody reads survives every future redeploy. foreach (var key in _trackers.Keys) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs index 632b0925..906c6da4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs @@ -664,6 +664,33 @@ public sealed class MqttSubscriptionManagerTests .ShouldBe(["f/oven/press", OvenTopic]); } + /// + /// R06:M-2. Register prunes the cache to the authored set — an unauthored reference's + /// BadNodeIdUnknown marker and a dropped tag's value go. Control: a still-authored tag keeps its + /// value across the re-register (a redeploy must not blank observed values). + /// + [Fact] + public async Task Register_PrunesCacheEntriesForReferencesNoLongerAuthored() + { + var mgr = NewManager(new FakeTransport()); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenTempPath, OvenPressPath, "Plant/Mqtt/broker1/Nope"], TimeSpan.Zero, TestContext.Current.CancellationToken); + mgr.HandleMessage(OvenTopic, "1"u8.ToArray(), retained: false); + mgr.HandleMessage("f/oven/press", "2"u8.ToArray(), retained: false); + mgr.Values.Read("Plant/Mqtt/broker1/Nope").StatusCode.ShouldBe(BadNodeIdUnknown); + + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + + mgr.Values.Read(OvenTempPath).Value.ShouldBe(1.0); + mgr.Values.Read(OvenPressPath).StatusCode.ShouldBe(0x80320000u); + mgr.Values.Read("Plant/Mqtt/broker1/Nope").StatusCode.ShouldBe(0x80320000u); + mgr.Values.Count.ShouldBe(1); + } + /// /// R06:M-11. A subscribe issued before any transport is attached reaches no broker, so it must /// not be recorded as established. It used to be — "subscribed, receives nothing" — and the first diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs index 1bd3b830..43c44103 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs @@ -852,6 +852,28 @@ public sealed class SparkplugIngestorTests ing.TryResolve(TempPath, out _).ShouldBeFalse(); } + /// + /// R06:M-2. Register prunes the shared cache to the authored set: an unauthored reference's + /// BadNodeIdUnknown marker and a dropped tag's value go, so the cache cannot grow with every RawPath + /// and stray reference ever seen. Control: a still-authored tag's value survives the re-register. + /// + [Fact] + public async Task Register_PrunesCacheEntriesForReferencesNoLongerAuthored() + { + var ing = NewIngestor(); + await ing.SubscribeAsync(["Plant/Mqtt/spb/NotAThing"], TimeSpan.Zero, TestContext.Current.CancellationToken); + var kept = new DataValueSnapshot(1.0f, 0u, null, DateTime.UtcNow); + ing.Values.Update(TempPath, kept); + ing.Values.Update(PressPath, kept); + + ing.Register([Tag(TempPath, Blob(Group, Node, Device, "Temperature", "Float32"))]); + + ing.Values.Read(TempPath).ShouldBe(kept); + ing.Values.Read(PressPath).StatusCode.ShouldBe(0x80320000u); // dropped tag: no stale value + ing.Values.Read("Plant/Mqtt/spb/NotAThing").StatusCode.ShouldBe(0x80320000u); + ing.Values.Count.ShouldBe(1); + } + [Fact] public async Task SubscribeAsync_UnauthoredReference_ReportsBadNodeIdUnknown() { From d67396cc6d7eaa6ecb8c15554509d0503a414b4d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:33:47 -0400 Subject: [PATCH 10/34] feat(mqtt): wire sparkplug.birthObservationWindowSeconds into the browse surface (R06:M-4) The authored window was persisted and consumed by nothing. It now (clamped 1-120 s) extends the Sparkplug OpenBudget, bounds a passive UntilStable-style settle on open (return after a 1 s quiet spell in NBIRTH/DBIRTH traffic), and bounds how long an operator rebirth request waits for the births it asked for before returning, so the re-rendered tree carries them. A caller deadline during the wait still reports the publish. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 4 +- .../MqttBrowseSession.cs | 80 +++++++++- .../MqttDriverBrowser.cs | 65 +++++++- .../MqttBirthObservationWindowTests.cs | 145 ++++++++++++++++++ 4 files changed, 284 insertions(+), 10 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index 8400ab42..d8620d65 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -117,7 +117,7 @@ driver are pure organisational grouping in the RawPath. "HostId": "", // Host Application identity — spBv1.0/STATE/{HostId} "ActAsPrimaryHost": false, // claim the primary host id (Primary-gated; see below) "RequestRebirthOnGap": true, // a seq gap asks the edge node to re-announce (NCMD) - "BirthObservationWindowSeconds": 15 // how long discovery collects births before calling it stable + "BirthObservationWindowSeconds": 15 // how long the browse picker collects births before calling them stable } } ``` @@ -128,7 +128,7 @@ driver are pure organisational grouping in the RawPath. | `HostId` | `""` | Sparkplug Host Application identity — the `spBv1.0/STATE/{HostId}` topic. Optional for observation; **required** to claim. Set without `ActAsPrimaryHost`, the driver only observes | | `ActAsPrimaryHost` | `false` | Claim the Sparkplug Primary Host Application identity. **Primary-gated** — only the node holding the redundancy Primary role claims it. See [Primary host](#primary-host-actasprimaryhost) | | `RequestRebirthOnGap` | `true` | On a detected sequence-number gap, publish a rebirth-request NCMD instead of running on stale metric state | -| `BirthObservationWindowSeconds` | `15` | Discovery's birth-collection window. Must be ≥ 1 | +| `BirthObservationWindowSeconds` | `15` | The Sparkplug browse picker's birth-collection window, clamped to 1–120 s (R06:M-4). The open budget grows by it; the open lingers while NBIRTH/DBIRTH traffic is still arriving (returning after a 1 s quiet spell — an idle plant costs 1 s); and an operator **Request rebirth** waits, bounded by it, for the births it asked for to arrive and go quiet for 500 ms, so the tree re-rendered afterwards carries them. A caller deadline during that wait still reports the publish. The runtime driver does not read it (its `DiscoverAsync` has no production caller — `SupportsOnlineDiscovery` is false) | `MqttDriverForm` authors all five in Sparkplug mode. The sub-object is **merged, never replaced** (a key a newer driver adds inside it survives an older AdminUI), and in `Plain` mode it is not touched at diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs index a231124b..9d4f6772 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs @@ -149,6 +149,9 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + /// The quiet spell that ends a rebirth request's wait once its births have started arriving. + private static readonly TimeSpan RebirthQuietInterval = TimeSpan.FromMilliseconds(500); + /// Bound on the courtesy DISCONNECT issued at dispose — teardown never hangs on a dead broker. private static readonly TimeSpan DisconnectTimeout = TimeSpan.FromSeconds(5); @@ -174,6 +177,7 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession private readonly ConcurrentQueue _publishedTopics = new(); private int _disposed; + private long _birthCount; private long _lastUsedTicksUtc = DateTime.UtcNow.Ticks; private int _nodeCount; private int _oversized; @@ -195,9 +199,16 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// The connected MQTT client; disconnected and disposed by . /// Optional logger; never receives credentials or payload bodies. /// Cap on recorded nodes; defaults to . - internal MqttBrowseSession(MqttMode mode, IMqttClient? client, ILogger? logger, int nodeCap = DefaultNodeCap) + /// + /// The Sparkplug birth-observation window: how long waits for the + /// births it asked for. (the test default) returns as soon as the NCMDs + /// are published. + /// + internal MqttBrowseSession( + MqttMode mode, IMqttClient? client, ILogger? logger, int nodeCap = DefaultNodeCap, TimeSpan birthWindow = default) { Mode = mode; + BirthWindow = birthWindow; _client = client; _logger = logger; _nodeCap = nodeCap; @@ -211,6 +222,56 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// internal int PublishCountForTest => Volatile.Read(ref _publishCount); + /// The Sparkplug birth-observation window this session was opened with (R06:M-4). + internal TimeSpan BirthWindow { get; } + + /// How many NBIRTH/DBIRTH messages this session has observed — the settle loop's signal. + internal long BirthCount => Interlocked.Read(ref _birthCount); + + /// + /// Waits until birth traffic has gone quiet for one interval, bounded by + /// — the UntilStable cadence of the birth-observation window. + /// + /// Reads the running birth count. + /// The count before the wait began. + /// + /// When true, quiet only counts once at least one birth has arrived past + /// — the shape a rebirth request needs, since its answer takes a broker round-trip to arrive. + /// + /// The quiet spell that ends the wait. + /// The hard ceiling; non-positive returns at once. + /// Cancels the wait. + /// A task that completes when the births have settled or the window has elapsed. + internal static async Task SettleBirthsAsync( + Func birthCount, + long baseline, + bool awaitFirstBirth, + TimeSpan quiet, + TimeSpan window, + CancellationToken cancellationToken) + { + if (window <= TimeSpan.Zero) return; + + var clock = System.Diagnostics.Stopwatch.StartNew(); + var last = baseline; + var sawBirth = !awaitFirstBirth; + while (clock.Elapsed < window) + { + var step = window - clock.Elapsed; + await Task.Delay(step < quiet ? step : quiet, cancellationToken).ConfigureAwait(false); + + var now = birthCount(); + if (now != last) + { + last = now; + sawBirth = true; + continue; + } + + if (sawBirth) return; + } + } + /// The topics this session published, in order — the discriminating view of the above. internal IReadOnlyList PublishedTopicsForTest => [.. _publishedTopics]; @@ -557,6 +618,8 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession if (!SparkplugTopic.TryParse(topic, out var parsed)) return; if (parsed.Type is not (SparkplugMessageType.NBIRTH or SparkplugMessageType.DBIRTH)) return; + Interlocked.Increment(ref _birthCount); + // The bound has to be on the whole message: unlike a plain payload, a birth must be parsed // entire to yield its metric list, and the parse runs on the shared dispatcher thread. if (payload.Length > MaxBirthPayloadBytes) @@ -898,6 +961,7 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession ArgumentException.ThrowIfNullOrWhiteSpace(scope); var targets = ResolveRebirthTargets(scope); + var baseline = BirthCount; foreach (var (groupId, edgeNodeId) in targets) { @@ -908,6 +972,20 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession .ConfigureAwait(false); } + // The birth-observation window (R06:M-4): wait — bounded — for the births just asked for to arrive + // and settle, so the tree the picker re-renders after this call already carries them. A courtesy, + // never a failure: the NCMDs are out, so a caller deadline here still reports the publish. + try + { + await SettleBirthsAsync( + () => BirthCount, baseline, awaitFirstBirth: true, RebirthQuietInterval, BirthWindow, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Published; the caller simply stopped waiting for the answer. + } + Touch(); return targets.Count; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs index 3ef6b659..8be8affc 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs @@ -39,6 +39,15 @@ public sealed class MqttDriverBrowser : IDriverBrowser /// Ceiling on the open budget — the picker must never hang on an unreachable broker. internal const int MaxOpenBudgetSeconds = 30; + /// Ceiling on the Sparkplug birth-observation window — an authored hour must not park a picker. + internal const int MaxBirthWindowSeconds = 120; + + /// + /// How long the open waits for a quiet spell in birth traffic before handing the session back. An + /// idle plant (births are never retained) costs exactly this, once. + /// + internal static readonly TimeSpan BirthQuietInterval = TimeSpan.FromSeconds(1); + /// Marks the transient browse identity in the broker's client-id/session logs. internal const string BrowseClientIdPrefix = "-browse-"; @@ -77,7 +86,8 @@ public sealed class MqttDriverBrowser : IDriverBrowser openCts.CancelAfter(OpenBudget(opts)); var client = new MqttClientFactory().CreateMqttClient(); - var session = new MqttBrowseSession(opts.Mode, client, _logger); + var birthWindow = opts.Mode == MqttMode.SparkplugB ? BirthWindow(opts) : TimeSpan.Zero; + var session = new MqttBrowseSession(opts.Mode, client, _logger, birthWindow: birthWindow); try { // Runs on MQTTnet's dispatcher: record and return, nothing else. @@ -100,6 +110,29 @@ public sealed class MqttDriverBrowser : IDriverBrowser .Build(); await client.SubscribeAsync(subscribeOptions, openCts.Token).ConfigureAwait(false); + if (opts.Mode == MqttMode.SparkplugB) + { + // The birth-observation window (R06:M-4): births already in flight — an edge node mid-rebirth, + // a primary-host flap — are collected until the traffic goes quiet, bounded by the window, so + // the first RootAsync is not a snapshot taken in the middle of a burst. Passive: nothing is + // published to provoke a birth. An idle plant costs one quiet interval. + try + { + await MqttBrowseSession.SettleBirthsAsync( + () => session.BirthCount, + baseline: session.BirthCount, + awaitFirstBirth: false, + BirthQuietInterval, + birthWindow, + openCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // The open budget ran out mid-settle: what has been observed is still a usable session. + } + } + _logger.LogInformation( "AdminUI MQTT browse session opened against {Host}:{Port} in {Mode} mode observing " + "'{Filter}' (read-only).", @@ -179,12 +212,30 @@ public sealed class MqttDriverBrowser : IDriverBrowser options with { CleanSession = true }; /// - /// Whole-open bound (connect + subscribe), clamped to - /// – around the config's - /// own connect deadline. + /// Whole-open bound: connect + subscribe, clamped to + /// – around the config's own + /// connect deadline — plus, in Sparkplug mode, the the open may spend + /// collecting births. /// /// The browse configuration. - /// The clamped open budget. - internal static TimeSpan OpenBudget(MqttDriverOptions options) => - TimeSpan.FromSeconds(Math.Clamp(options.ConnectTimeoutSeconds, MinOpenBudgetSeconds, MaxOpenBudgetSeconds)); + /// The open budget. + internal static TimeSpan OpenBudget(MqttDriverOptions options) + { + var connect = TimeSpan.FromSeconds( + Math.Clamp(options.ConnectTimeoutSeconds, MinOpenBudgetSeconds, MaxOpenBudgetSeconds)); + return options.Mode == MqttMode.SparkplugB ? connect + BirthWindow(options) : connect; + } + + /// + /// The authored sparkplug.birthObservationWindowSeconds, clamped to 1– + /// s: how long the browse surface collects NBIRTH/DBIRTH traffic before calling the observed metric set + /// stable (on open, and after an operator rebirth request). + /// + /// The browse configuration. + /// The clamped window. + internal static TimeSpan BirthWindow(MqttDriverOptions options) => + TimeSpan.FromSeconds(Math.Clamp( + options.Sparkplug?.BirthObservationWindowSeconds ?? new MqttSparkplugOptions().BirthObservationWindowSeconds, + 1, + MaxBirthWindowSeconds)); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs new file mode 100644 index 00000000..b7f920af --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs @@ -0,0 +1,145 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser; +using SparkplugDataType = Org.Eclipse.Tahu.Protobuf.DataType; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; + +/// +/// R06:M-4. sparkplug.birthObservationWindowSeconds was authored, range-checked and +/// persisted by the AdminUI and consumed by nothing. It now bounds how long the Sparkplug browse surface +/// collects NBIRTH/DBIRTH traffic before calling the observed set stable: the open budget grows by it, +/// the open lingers while births are still arriving, and an operator rebirth request waits (bounded) for +/// the births it asked for, so the tree the picker re-renders already carries them. +/// +public sealed class MqttBirthObservationWindowTests +{ + private static MqttDriverOptions Sparkplug(int windowSeconds, int connectTimeoutSeconds = 10) => new() + { + Mode = MqttMode.SparkplugB, + ConnectTimeoutSeconds = connectTimeoutSeconds, + Sparkplug = new MqttSparkplugOptions { GroupId = "Plant1", BirthObservationWindowSeconds = windowSeconds }, + }; + + [Fact] + public void OpenBudget_Sparkplug_AddsTheBirthObservationWindow() + { + MqttDriverBrowser.OpenBudget(Sparkplug(windowSeconds: 15)).ShouldBe(TimeSpan.FromSeconds(10 + 15)); + MqttDriverBrowser.OpenBudget(Sparkplug(windowSeconds: 40)).ShouldBe(TimeSpan.FromSeconds(10 + 40)); + } + + [Fact] + public void OpenBudget_Plain_IgnoresTheSparkplugWindow() + { + var plain = Sparkplug(windowSeconds: 40) with { Mode = MqttMode.Plain }; + MqttDriverBrowser.OpenBudget(plain).ShouldBe(TimeSpan.FromSeconds(10)); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(-5, 1)] + [InlineData(15, 15)] + [InlineData(100_000, 120)] + public void BirthWindow_IsClamped(int configured, int expectedSeconds) + => MqttDriverBrowser.BirthWindow(Sparkplug(configured)).ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + + /// An idle plant costs one quiet interval, not the whole window. + [Fact] + public async Task Settle_WithNoBirths_ReturnsAfterOneQuietInterval() + { + var reads = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => { reads++; return 0; }, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromMinutes(5), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + reads.ShouldBe(1); + } + + /// + /// While births keep arriving the settle keeps collecting; it returns on the first quiet interval + /// after the last one. Counted in reads, not wall-clock, so it cannot flake. + /// + [Fact] + public async Task Settle_KeepsCollectingWhileBirthsArrive_ThenReturnsOnQuiet() + { + var reads = 0; + long births = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => + { + reads++; + if (reads <= 5) births++; // five intervals with fresh births, then silence + return births; + }, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromMinutes(5), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + reads.ShouldBe(6); + } + + /// The window is a hard ceiling: a birth flood that never goes quiet still ends. + [Trait("Category", "SlowUnit")] + [Fact] + public async Task Settle_NeverQuiet_EndsAtTheWindow() + { + long births = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => ++births, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromSeconds(1), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + births.ShouldBeGreaterThan(1); + } + + /// + /// A rebirth request waits for the birth it asked for. The fixed version cannot complete + /// synchronously (it awaits the settle), the unfixed one completes before the call returns — so the + /// IsCompleted check is deterministic, not a timing race. The birth then releases it. + /// + [Fact] + public async Task RequestRebirth_WaitsForTheRequestedBirths_ThenReturns() + { + await using var s = new MqttBrowseSession( + MqttMode.SparkplugB, client: null, logger: null, birthWindow: TimeSpan.FromMinutes(5)); + + var request = s.RequestRebirthAsync("Plant1/EdgeA", TestContext.Current.CancellationToken); + request.IsCompleted.ShouldBeFalse("the request must wait for the births it asked for"); + + s.ObserveBirthForTest("Plant1", "EdgeA", null, "Uptime", SparkplugDataType.Int64); + + (await request.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBe(1); + s.PublishCountForTest.ShouldBe(1); + } + + /// + /// The wait is a courtesy, never a failure: the NCMD was already published, so a caller deadline + /// during the settle returns the published count instead of reporting an error for a request that + /// succeeded. + /// + [Fact] + public async Task RequestRebirth_CallerCancelsDuringTheSettle_StillReportsThePublish() + { + await using var s = new MqttBrowseSession( + MqttMode.SparkplugB, client: null, logger: null, birthWindow: TimeSpan.FromMinutes(5)); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + var request = s.RequestRebirthAsync("Plant1/EdgeA", cts.Token); + request.IsCompleted.ShouldBeFalse(); + await cts.CancelAsync(); + + (await request.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBe(1); + } +} From c14a3388f1d83b2e2c3fee52603b6533fa7e7e8d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:35:55 -0400 Subject: [PATCH 11/34] fix(mqtt): Sparkplug array and unsupported-type metrics are not bindable from the browser (R06:M-5) The ingestor refuses array metrics and unmapped types, yet DescribeMetric still stated their binding tuple, so a browse-commit produced a tag stuck at BadWaitingForInitialData forever. Those leaves now state no address (the commit refuses them) and say why in the descriptor line. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 5 ++++- .../MqttBrowseSession.cs | 13 +++++++++++-- .../MqttBrowseSessionTests.cs | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index d8620d65..b9e62582 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -249,7 +249,10 @@ One subscription (`spBv1.0/{GroupId}/#`) feeds a per-edge-node state machine [Structured metrics](#structured-metrics-template--dataset--propertyset). - **Unsupported datatypes are skipped with a warning, never guessed**: `PropertySetList`, `Unknown`, extension values, and any `Template`/`DataSet` that cannot be flattened honestly. `Int8`→`Int16` and - `UInt8`→`UInt16` widen; `*Array` variants map to the element type with the array bit set. + `UInt8`→`UInt16` widen; `*Array` variants map to the element type with the array bit set — but + **array metrics are not read in v1** (warned once, never fed), so the browse picker offers them, and any + metric whose type is unsupported, **without an address**: the browse-commit refuses them rather than + committing a tag that waits for initial data forever (R06:M-5). - **Ingest state is reset at every session/authoring seam** — a reconnect or a `ReinitializeAsync` must not carry a previous session's alias table into a new one. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs index 9d4f6772..b6bf8d9c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs @@ -441,16 +441,25 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession var mapped = metric.DataType?.ToDriverDataType(); var sparkplugName = metric.DataType?.ToString() ?? nameof(SparkplugDataType.Unknown); + var isArray = metric.DataType?.IsSparkplugArray() ?? false; + var descriptor = new StringBuilder(metric.BirthType.ToString()).Append(" · ").Append(sparkplugName); if (metric.Alias is { } alias) descriptor.Append(" · alias ").Append(alias); if (mapped is null) descriptor.Append(" · unsupported in v1 — not bindable"); + else if (isArray) descriptor.Append(" · array metrics are not read in v1 — not bindable"); + + // Unbindable = the ingestor refuses it: an unmapped type, or ANY array variant (SparkplugIngestor + // gates on the array bit and never feeds the tag). Such a leaf states NO address, so the + // browse-commit refuses it at commit instead of writing a tag that sits at + // BadWaitingForInitialData forever (R06:M-5). Labelling it alone was not enough — it still bound. + var bindable = mapped is not null && !isArray; return new AttributeInfo( metric.Name, mapped?.ToString() ?? sparkplugName, - metric.DataType?.IsSparkplugArray() ?? false, + isArray, descriptor.ToString(), - AddressFields: BuildMetricAddressFields(metric.Name, scope)); + AddressFields: bindable ? BuildMetricAddressFields(metric.Name, scope) : null); } /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs index bbc14bc1..fd624dde 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs @@ -779,6 +779,13 @@ public sealed class MqttBrowseSessionTests "Plant1/EdgeA::Profile", TestContext.Current.CancellationToken)).ShouldHaveSingleItem(); a.DriverDataType.ShouldBe(nameof(Core.Abstractions.DriverDataType.Int32)); // element type a.IsArray.ShouldBeTrue(); + + // R06:M-5: the ingestor refuses array metrics ("not supported in v1"), so the browser must not + // offer one as bindable — no stated address means the browse-commit refuses it in words, instead + // of committing a tag that sits at BadWaitingForInitialData forever. The scalar control is + // SparkplugAttributes_StateTheBindingTuple_SoTheCommitNeverParsesTheNodeId. + a.AddressFields.ShouldBeNull(); + a.SecurityClass.ShouldContain("not bindable"); } [Fact] @@ -796,6 +803,7 @@ public sealed class MqttBrowseSessionTests Enum.TryParse(a.DriverDataType, ignoreCase: true, out _) .ShouldBeFalse(); // not a guessed type — unmappable, and visibly so a.SecurityClass.ShouldContain("unsupported"); + a.AddressFields.ShouldBeNull(); // R06:M-5 — labelled AND unbindable, not just labelled } [Fact] From 4a3d7a9361f440b51937ecbf7cd18e7667e8b050 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:35:58 -0400 Subject: [PATCH 12/34] feat(modbus): wire Family / MelsecSubFamily / DisableFC23, and the form's timing keys U1: a tag's addressString (Modicon / mnemonic / DL205 V-memory / MELSEC D-X-Y grammar) now resolves through ModbusAddressParser with the DRIVER's Family and MelsecSubFamily at every initialize and is authoritative; the family-unaware gates (deploy Inspect, AdminUI parse consult) accept what some family resolves. The tag editor gets an address-text field that resolves with the owning driver's family and persists both the string and the resolved region/offset/type fields; a hand edit drops the string. DisableFC23 now gates FC23: by default a 2..121-register holding write is one FC23 write + read-back (a unit answering exception 01, or failing FC23 at the transport, moves to FC16 for the rest of the generation); RTU framing deframes the FC23 reply. Live-checked: the pymodbus fixture implements FC23. Also: the AdminUI driver form serialises ModbusDriverOptions (TimeSpan strings), which the factory DTO never read - every timing field on the form was dead. The DTO now accepts both shapes (*Ms keys win). Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriverOptions.cs | 20 +- .../ModbusTagDefinitionFactory.cs | 148 ++++++++++++--- .../ModbusDriver.cs | 77 +++++++- .../ModbusDriverFactoryExtensions.cs | 53 +++++- .../ModbusRtuFraming.cs | 11 +- .../Drivers/Forms/ModbusDriverForm.razor | 2 +- .../TagEditors/ModbusTagConfigEditor.razor | 49 ++++- .../Uns/TagEditors/ModbusTagConfigModel.cs | 73 ++++++- .../CommsRecoveryTests.cs | 31 +++ .../ModbusDriverTests.cs | 21 ++ .../ModbusFamilyWiringTests.cs | 179 ++++++++++++++++++ .../ModbusRtuFramingTests.cs | 11 ++ .../Uns/ModbusTagConfigModelTests.cs | 39 ++++ 13 files changed, 656 insertions(+), 58 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs index 9d6b5479..4f456585 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs @@ -89,10 +89,13 @@ public sealed class ModbusDriverOptions public bool UseFC16ForSingleRegisterWrites { get; init; } = false; /// - /// Reserved / no-op kill-switch for FC23 (Read/Write Multiple Registers). The - /// driver does not currently emit FC23 — toggling this option has no observable effect - /// today. The slot exists so a future block-read-coalescing enhancement that opts into - /// FC23 can be disabled per-deployment without a code change. Default false. + /// Kill-switch for FC23 (Read/Write Multiple Registers). By default a multi-register holding + /// write (2–121 registers: 32/64-bit values, arrays, strings) is sent as ONE FC23 that writes + /// the block and reads the same block back, so the device confirms what it now holds. A unit + /// that answers FC23 with exception 01 (MELSEC QJ71MT91, S7 Modbus servers) is moved to FC16 + /// automatically, for that write and every later one; one that fails FC23 at the transport is + /// moved too (that one write is reported as a comms failure). Set true to always use + /// FC16. Default false. /// public bool DisableFC23 { get; init; } = false; @@ -128,10 +131,11 @@ public sealed class ModbusDriverOptions public bool DisableReadCoalescing { get; init; } = false; /// - /// PLC family hint that drives the parser's family-native branch. When set to a - /// non-Generic value, address strings using that family's native syntax (DL205 V2000 / - /// MELSEC D100) parse to the right region + offset directly. Defaults to - /// = Modicon-only behaviour preserved by default. + /// PLC family hint that drives the parser's family-native branch. A tag authored with an + /// addressString is resolved with this family at every initialize, so with a non-Generic + /// value that family's native syntax (DL205 V2000 octal / MELSEC D100) parses to the right + /// region + offset. Defaults to (Modicon / mnemonic grammar). + /// Tags authored by region + numeric address are unaffected. /// public ModbusFamily Family { get; init; } = ModbusFamily.Generic; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs index 86b0a4e9..d34c55ba 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs @@ -32,6 +32,24 @@ public static class ModbusTagDefinitionFactory /// The mapped definition when this returns . /// when is a valid Modbus address object. public static bool FromTagConfig(string tagConfig, string rawPath, out ModbusTagDefinition def) + => FromTagConfig(tagConfig, rawPath, ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out def); + + /// + /// Family-aware overload: an addressString key (the Modicon / mnemonic / family-native grammar + /// of — 40001:F:CDAB, DL205 V2000, MELSEC D100) + /// is resolved with the DRIVER's / and is + /// authoritative for region, offset, type, bit, string length, byte order and array count; the + /// editor also persists the resolved region / address keys beside it for display and for + /// readers that predate it. Without an addressString this is exactly the 3-argument overload. + /// + /// The authored equipment-tag TagConfig JSON. + /// The tag's RawPath — becomes the definition's identity (Name). + /// The driver's PLC family (from its DriverConfig). + /// The driver's MELSEC sub-family; consulted only for MELSEC. + /// The mapped definition when this returns . + /// when is a valid Modbus address object. + public static bool FromTagConfig(string tagConfig, string rawPath, ModbusFamily family, MelsecFamily melsecSubFamily, + out ModbusTagDefinition def) { def = null!; if (string.IsNullOrWhiteSpace(tagConfig)) return false; @@ -39,28 +57,60 @@ public static class ModbusTagDefinitionFactory { using var doc = JsonDocument.Parse(tagConfig); var root = doc.RootElement; - if (root.ValueKind != JsonValueKind.Object - || !root.TryGetProperty("region", out _) - || !root.TryGetProperty("address", out var addr)) - return false; - if (addr.ValueKind != JsonValueKind.Number - || !addr.TryGetInt32(out var address) - || address < 0 || address > ushort.MaxValue) - return false; - // Strict enum reads: a present-but-invalid (typo'd) enum rejects the tag - // (→ BadNodeIdUnknown) instead of silently defaulting to a wrong-width Good. - if (!TagConfigJson.TryReadEnumStrict(root, "region", ModbusRegion.HoldingRegisters, out var region)) return false; - // An absent dataType defaults per region: a bit region (Coils / DiscreteInputs) only carries - // Bool, so defaulting it to Int16 would reject every bit tag authored without the key. - if (!TagConfigJson.TryReadEnumStrict(root, "dataType", DefaultDataType(region), out var dataType)) return false; + if (root.ValueKind != JsonValueKind.Object) return false; + + ModbusRegion region; + int address; + ModbusDataType dataType; + ModbusByteOrder byteOrder; + byte bitIndex; + ushort stringLength; + int? arrayCount; + var addressText = ReadAddressString(root, out var addressStringInvalid); + if (addressStringInvalid) return false; + if (addressText is not null) + { + if (!ModbusAddressParser.TryParse(addressText, family, melsecSubFamily, out var parsed, out _) || parsed is null) + return false; + region = parsed.Region; + address = parsed.Offset; + dataType = parsed.DataType; + byteOrder = parsed.ByteOrder; + bitIndex = parsed.Bit ?? 0; + stringLength = parsed.StringLength; + arrayCount = parsed.ArrayCount; + } + else + { + if (!root.TryGetProperty("region", out _) || !root.TryGetProperty("address", out var addr)) + return false; + if (addr.ValueKind != JsonValueKind.Number + || !addr.TryGetInt32(out address) + || address < 0 || address > ushort.MaxValue) + return false; + // Strict enum reads: a present-but-invalid (typo'd) enum rejects the tag + // (→ BadNodeIdUnknown) instead of silently defaulting to a wrong-width Good. + if (!TagConfigJson.TryReadEnumStrict(root, "region", ModbusRegion.HoldingRegisters, out region)) return false; + // An absent dataType defaults per region: a bit region (Coils / DiscreteInputs) only carries + // Bool, so defaulting it to Int16 would reject every bit tag authored without the key. + if (!TagConfigJson.TryReadEnumStrict(root, "dataType", DefaultDataType(region), out dataType)) return false; + if (!TagConfigJson.TryReadEnumStrict(root, "byteOrder", ModbusByteOrder.BigEndian, out byteOrder)) return false; + bitIndex = (byte)ReadInt(root, "bitIndex"); + stringLength = (ushort)ReadInt(root, "stringLength"); + // isArray / arrayLength — optional keys authored by the typed Modbus tag editor. + // Canonical rule: a tag is an array iff isArray:true AND arrayLength >= 1. + // isArray:false (with any arrayLength) is scalar — the foundation materialises a + // scalar OPC UA node, so returning an array value would cause a shape mismatch. + // A 1-element array (isArray:true, arrayLength:1) IS valid and reads as short[1]. + var isArray = ReadBool(root, "isArray"); + var arrayLength = ReadInt(root, "arrayLength"); + arrayCount = (isArray && arrayLength >= 1) ? arrayLength : null; + } // Region/type compatibility: a bit region carries only Bool, a register region never Bool. // A mismatch used to throw out of the coalescing planner (killing every tag's poll) or // silently return a bool into an Int32 node. if (!IsRegionTypeCompatible(region, dataType)) return false; - if (!TagConfigJson.TryReadEnumStrict(root, "byteOrder", ModbusByteOrder.BigEndian, out var byteOrder)) return false; if (!TagConfigJson.TryReadEnumStrict(root, "stringByteOrder", ModbusStringByteOrder.HighByteFirst, out var stringByteOrder)) return false; - var bitIndex = (byte)ReadInt(root, "bitIndex"); - var stringLength = (ushort)ReadInt(root, "stringLength"); // Guard: String tags require StringLength >= 1. RegisterCount = (StringLength+1)/2, // so StringLength=0 → 0 registers → spec-illegal FC03/FC04 with quantity=0 → PLC // returns exception 03. Reject here so the driver surfaces BadNodeIdUnknown rather @@ -69,14 +119,6 @@ public static class ModbusTagDefinitionFactory // Guard: BitInRegister tags require BitIndex 0–15. Shift by ≥ 16 on an int overflows // a ushort mask silently — reads always return false, writes have no effect. if (dataType == ModbusDataType.BitInRegister && bitIndex > 15) return false; - // isArray / arrayLength — optional keys authored by the typed Modbus tag editor. - // Canonical rule: a tag is an array iff isArray:true AND arrayLength >= 1. - // isArray:false (with any arrayLength) is scalar — the foundation materialises a - // scalar OPC UA node, so returning an array value would cause a shape mismatch. - // A 1-element array (isArray:true, arrayLength:1) IS valid and reads as short[1]. - var isArray = ReadBool(root, "isArray"); - var arrayLength = ReadInt(root, "arrayLength"); - int? arrayCount = (isArray && arrayLength >= 1) ? arrayLength : null; // "writable" defaults to true when absent (today's value) — makes read-only equipment tags // authorable. Node-level authz remains the effective write gate. var writable = TagConfigJson.ReadWritable(root); @@ -100,6 +142,42 @@ public static class ModbusTagDefinitionFactory catch (InvalidOperationException) { return false; } } + /// + /// Family-unaware acceptance, for the gates that do not know the owning driver's family (the deploy + /// gate's , CSV import, the AdminUI parse consult): accepts a blob SOME family + /// resolves, so the gate is never stricter than the driver it guards. Blobs without an + /// addressString behave exactly like . + /// + /// The authored equipment-tag TagConfig JSON. + /// The tag's RawPath. + /// The definition under the first family that resolves it. + /// when any family's driver would serve the tag. + public static bool FromTagConfigAnyFamily(string tagConfig, string rawPath, out ModbusTagDefinition def) + { + foreach (var (family, sub) in AllFamilies) + if (FromTagConfig(tagConfig, rawPath, family, sub, out def)) return true; + def = null!; + return false; + } + + private static readonly (ModbusFamily Family, MelsecFamily Sub)[] AllFamilies = + [ + (ModbusFamily.Generic, MelsecFamily.Q_L_iQR), + (ModbusFamily.DL205, MelsecFamily.Q_L_iQR), + (ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR), + (ModbusFamily.MELSEC, MelsecFamily.F_iQF), + ]; + + /// Reads the optional addressString: null when absent or blank; flags a non-string value. + private static string? ReadAddressString(JsonElement root, out bool invalid) + { + invalid = false; + if (!root.TryGetProperty("addressString", out var e) || e.ValueKind == JsonValueKind.Null) return null; + if (e.ValueKind != JsonValueKind.String) { invalid = true; return null; } + var text = e.GetString(); + return string.IsNullOrWhiteSpace(text) ? null : text.Trim(); + } + /// The data type an authored tag gets when its dataType key is absent. /// The tag's region. /// Bool for Coils / DiscreteInputs, Int16 for the register regions. @@ -194,7 +272,7 @@ public static class ModbusTagDefinitionFactory // The accept gate: whatever the driver reads draws nothing. See the remarks. The RAW reference is // handed to the mapper — the driver reads the stored blob, not a trimmed copy of it. - if (FromTagConfig(reference, InspectionRawPath, out _)) return warnings; + if (FromTagConfigAnyFamily(reference, InspectionRawPath, out _)) return warnings; try { @@ -235,6 +313,24 @@ public static class ModbusTagDefinitionFactory /// The list to append warnings to. private static void DescribeRejection(JsonElement root, List warnings) { + var addressText = ReadAddressString(root, out var addressStringInvalid); + if (addressStringInvalid) + { + warnings.Add("Modbus TagConfig 'addressString' is not a string — the tag will not resolve (BadNodeIdUnknown)."); + return; + } + if (addressText is not null) + { + // The addressString is authoritative; the other address keys are not read at all. + if (!ModbusAddressParser.TryParse(addressText, out _, out var error)) + warnings.Add($"Modbus 'addressString' '{addressText}' does not parse under any PLC family ({error}) — " + + "the tag will not resolve (BadNodeIdUnknown)."); + else + warnings.Add($"Modbus 'addressString' '{addressText}' resolves to an unusable tag (check its type against " + + "the region) — the tag will not resolve (BadNodeIdUnknown)."); + return; + } + if (!root.TryGetProperty("region", out _)) { warnings.Add( diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index 4aa312d3..2bc8e8ef 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -533,7 +533,7 @@ public sealed class ModbusDriver _skippedTags.BeginRebuild(_options.RawTags.Count); foreach (var entry in _options.RawTags) { - if (ModbusTagDefinitionFactory.FromTagConfig(entry.TagConfig, entry.RawPath, out var def)) + if (ModbusTagDefinitionFactory.FromTagConfig(entry.TagConfig, entry.RawPath, _options.Family, _options.MelsecSubFamily, out var def)) table[entry.RawPath] = def with { WriteIdempotent = entry.WriteIdempotent }; else { @@ -556,6 +556,7 @@ public sealed class ModbusDriver _transport = transport; Volatile.Write(ref _tagsByRawPath, table); Volatile.Write(ref _refusalWarned, new ConcurrentDictionary(StringComparer.Ordinal)); + Volatile.Write(ref _fc23Unsupported, new ConcurrentDictionary()); WriteHealth(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); // PR 23: kick off the probe loop once the transport is up. Initial state stays @@ -1619,6 +1620,7 @@ public sealed class ModbusDriver throw new ModbusWriteRefusedException(StatusBadOutOfRange, $"Write of {qty} registers to {tag.Name} exceeds MaxRegistersPerWrite={writeCap}. " + $"Split the tag (e.g. shorter StringLength or smaller ArrayCount) — partial FC16 chunks would lose atomicity."); + if (await TryWriteViaFc23Async(transport, tag, bytes, qty, ct).ConfigureAwait(false)) return; var pdu = new byte[6 + 1 + bytes.Length]; pdu[0] = 0x10; pdu[1] = (byte)(tag.Address >> 8); pdu[2] = (byte)(tag.Address & 0xFF); @@ -1634,6 +1636,79 @@ public sealed class ModbusDriver } } + /// FC23's spec ceiling on the write quantity (0x79); larger multi-register writes use FC16. + private const ushort Fc23MaxWriteRegisters = 121; + + /// + /// Units that refused FC23 (exception 01) or failed it at the transport; their multi-register writes + /// use FC16 for the rest of this driver generation. Replaced (never cleared in place) on re-init. + /// + private ConcurrentDictionary _fc23Unsupported = new(); + + /// + /// Multi-register write via FC23 (Read/Write Multiple Registers): the device writes the block and + /// returns the SAME block read back in one transaction, so the write is confirmed by the device's own + /// post-write registers. Gated by . A device that does + /// not implement FC23 (MELSEC QJ71MT91, S7 Modbus servers — exception 01) falls back to FC16 for this + /// write and every later one on that unit, which is safe because an Illegal Function wrote nothing. + /// A transport failure on FC23 is reported (its outcome is unknown, so it is not retried as FC16) and + /// also moves the unit to FC16, so a gateway that cannot frame FC23 costs one failed write, not all. + /// A read-back that differs from what was written (PLC logic overwrote it in the same scan, or the + /// device clamped it) is logged once per tag; the write itself was accepted. + /// + /// when FC23 performed the write; to use FC16. + private async Task TryWriteViaFc23Async( + IModbusTransport transport, ModbusTagDefinition tag, byte[] bytes, ushort qty, CancellationToken ct) + { + var unit = ResolveUnitId(tag); + var unsupported = Volatile.Read(ref _fc23Unsupported); + // Only genuinely multi-register writes: a single register forced onto FC16 (UseFC16ForSingleRegisterWrites) + // stays on the function code the operator named. + if (_options.DisableFC23 || qty < 2 || qty > Fc23MaxWriteRegisters || unsupported.ContainsKey(unit)) return false; + + var pdu = new byte[10 + 1 + bytes.Length]; + pdu[0] = 0x17; + pdu[1] = (byte)(tag.Address >> 8); pdu[2] = (byte)(tag.Address & 0xFF); // read start + pdu[3] = (byte)(qty >> 8); pdu[4] = (byte)(qty & 0xFF); // read quantity + pdu[5] = (byte)(tag.Address >> 8); pdu[6] = (byte)(tag.Address & 0xFF); // write start + pdu[7] = (byte)(qty >> 8); pdu[8] = (byte)(qty & 0xFF); // write quantity + pdu[9] = (byte)bytes.Length; + Buffer.BlockCopy(bytes, 0, pdu, 10, bytes.Length); + + byte[] resp; + try + { + resp = await transport.SendAsync(unit, pdu, ct).ConfigureAwait(false); + } + catch (ModbusException mex) when (mex.ExceptionCode == 0x01) + { + if (unsupported.TryAdd(unit, 0)) + _logger.LogInformation( + "Modbus unit {Unit} does not implement FC23 (Illegal Function); multi-register writes use FC16. Driver={DriverInstanceId}", + unit, _driverInstanceId); + return false; + } + catch (Exception ex) when (ex is not ModbusException && !(ex is OperationCanceledException && ct.IsCancellationRequested)) + { + if (unsupported.TryAdd(unit, 0)) + _logger.LogWarning(ex, + "Modbus FC23 write to unit {Unit} failed at the transport; later multi-register writes to it use FC16. Driver={DriverInstanceId}", + unit, _driverInstanceId); + throw; + } + + // resp = [0x17][byteCount][read-back registers…] + if (resp.Length < 2 || resp[1] != bytes.Length || resp.Length < 2 + resp[1]) + throw new InvalidDataException( + $"Modbus FC23 response for {tag.Name} malformed: expected {bytes.Length} read-back bytes, got {(resp.Length < 2 ? -1 : resp[1])}"); + if (!resp.AsSpan(2, bytes.Length).SequenceEqual(bytes) + && Volatile.Read(ref _refusalWarned).TryAdd("fc23:" + tag.Name, 0)) + _logger.LogWarning( + "Modbus FC23 read-back of {RawPath} differs from the value written (device logic or clamping changed it); the write was accepted. Driver={DriverInstanceId}", + tag.Name, _driverInstanceId); + return true; + } + /// /// Encode an array-typed write value into a contiguous byte block by encoding each /// element with the scalar codec. Caller submits IList / Array of the element CLR type. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs index 9aa8e0d7..0886124a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs @@ -78,7 +78,7 @@ public static class ModbusDriverFactoryExtensions Host = dto.Host!, Port = dto.Port ?? 502, UnitId = dto.UnitId ?? 1, - Timeout = TimeSpan.FromMilliseconds(dto.TimeoutMs ?? 2_000), + Timeout = Span(dto.TimeoutMs, dto.Timeout) ?? TimeSpan.FromSeconds(2), MaxRegistersPerRead = dto.MaxRegistersPerRead ?? 125, MaxRegistersPerWrite = dto.MaxRegistersPerWrite ?? 123, MaxCoilsPerRead = dto.MaxCoilsPerRead ?? 2000, @@ -94,7 +94,7 @@ public static class ModbusDriverFactoryExtensions : ParseEnum(dto.MelsecSubFamily, "", driverInstanceId, "MelsecSubFamily"), Transport = dto.Transport is null ? ModbusTransportMode.Tcp : ParseEnum(dto.Transport, "", driverInstanceId, "Transport"), - AutoProhibitReprobeInterval = dto.AutoProhibitReprobeMs is { } reprobeMs ? TimeSpan.FromMilliseconds(reprobeMs) : null, + AutoProhibitReprobeInterval = Span(dto.AutoProhibitReprobeMs, dto.AutoProhibitReprobeInterval), AutoReconnect = dto.AutoReconnect ?? true, // v3: the deploy artifact (Wave C) populates rawTags with the cluster's authored raw Modbus // tags. The driver maps each RawTagEntry's TagConfig blob through ModbusTagDefinitionFactory @@ -104,22 +104,22 @@ public static class ModbusDriverFactoryExtensions Probe = new ModbusProbeOptions { Enabled = dto.Probe?.Enabled ?? true, - Interval = TimeSpan.FromMilliseconds(dto.Probe?.IntervalMs ?? 5_000), - Timeout = TimeSpan.FromMilliseconds(dto.Probe?.TimeoutMs ?? 2_000), + Interval = Span(dto.Probe?.IntervalMs, dto.Probe?.Interval) ?? TimeSpan.FromSeconds(5), + Timeout = Span(dto.Probe?.TimeoutMs, dto.Probe?.Timeout) ?? TimeSpan.FromSeconds(2), ProbeAddress = dto.Probe?.ProbeAddress ?? 0, }, KeepAlive = dto.KeepAlive is null ? new ModbusKeepAliveOptions() : new ModbusKeepAliveOptions { Enabled = dto.KeepAlive.Enabled ?? true, - Time = TimeSpan.FromMilliseconds(dto.KeepAlive.TimeMs ?? 30_000), - Interval = TimeSpan.FromMilliseconds(dto.KeepAlive.IntervalMs ?? 10_000), + Time = Span(dto.KeepAlive.TimeMs, dto.KeepAlive.Time) ?? TimeSpan.FromSeconds(30), + Interval = Span(dto.KeepAlive.IntervalMs, dto.KeepAlive.Interval) ?? TimeSpan.FromSeconds(10), RetryCount = dto.KeepAlive.RetryCount ?? 3, }, - IdleDisconnectTimeout = dto.IdleDisconnectMs is { } ms ? TimeSpan.FromMilliseconds(ms) : null, + IdleDisconnectTimeout = Span(dto.IdleDisconnectMs, dto.IdleDisconnectTimeout), Reconnect = dto.Reconnect is null ? new ModbusReconnectOptions() : new ModbusReconnectOptions { - InitialDelay = TimeSpan.FromMilliseconds(dto.Reconnect.InitialDelayMs ?? 0), - MaxDelay = TimeSpan.FromMilliseconds(dto.Reconnect.MaxDelayMs ?? 30_000), + InitialDelay = Span(dto.Reconnect.InitialDelayMs, dto.Reconnect.InitialDelay) ?? TimeSpan.Zero, + MaxDelay = Span(dto.Reconnect.MaxDelayMs, dto.Reconnect.MaxDelay) ?? TimeSpan.FromSeconds(30), BackoffMultiplier = dto.Reconnect.BackoffMultiplier ?? 2.0, }, }; @@ -127,6 +127,23 @@ public static class ModbusDriverFactoryExtensions return options; } + /// + /// One timing knob, accepted in BOTH shapes the config can carry: the *Ms integer keys (CLI, + /// seed data, hand-authored JSON) and the TimeSpan-string keys the AdminUI driver form emits — it + /// serialises itself, so it writes "timeout":"00:00:07", not + /// timeoutMs. Before this every timing field on that form was silently ignored. The + /// millisecond key wins when both are present; an unparseable TimeSpan string is ignored (the + /// previous behaviour for these keys), never a reason to fail the driver. + /// + private static TimeSpan? Span(int? milliseconds, JsonElement? timeSpan) + { + if (milliseconds is { } ms) return TimeSpan.FromMilliseconds(ms); + if (timeSpan is { ValueKind: JsonValueKind.String } e + && TimeSpan.TryParse(e.GetString(), System.Globalization.CultureInfo.InvariantCulture, out var parsed)) + return parsed; + return null; + } + private static T ParseEnum(string? raw, string? tagName, string driverInstanceId, string field) where T : struct, Enum { if (string.IsNullOrWhiteSpace(raw)) @@ -156,6 +173,8 @@ public static class ModbusDriverFactoryExtensions public byte? UnitId { get; init; } /// Gets or sets the timeout in milliseconds. public int? TimeoutMs { get; init; } + /// Gets or sets the timeout as a TimeSpan string (the AdminUI form's shape). + public JsonElement? Timeout { get; init; } /// Gets or sets the maximum number of registers per read operation. public ushort? MaxRegistersPerRead { get; init; } /// Gets or sets the maximum number of registers per write operation. @@ -182,6 +201,8 @@ public static class ModbusDriverFactoryExtensions public string? Transport { get; init; } /// Gets or sets the automatic prohibition reprobei interval in milliseconds. public int? AutoProhibitReprobeMs { get; init; } + /// Gets or sets the re-probe interval as a TimeSpan string (the AdminUI form's shape). + public JsonElement? AutoProhibitReprobeInterval { get; init; } /// Gets or sets a value indicating whether automatic reconnection is enabled. public bool? AutoReconnect { get; init; } /// Gets or sets the authored raw tags (RawPath + TagConfig blob + WriteIdempotent) the @@ -194,6 +215,8 @@ public static class ModbusDriverFactoryExtensions public ModbusKeepAliveDto? KeepAlive { get; init; } /// Gets or sets the idle disconnect timeout in milliseconds. public int? IdleDisconnectMs { get; init; } + /// Gets or sets the idle disconnect timeout as a TimeSpan string (the AdminUI form's shape). + public JsonElement? IdleDisconnectTimeout { get; init; } /// Gets or sets the reconnect configuration. public ModbusReconnectDto? Reconnect { get; init; } } @@ -206,6 +229,10 @@ public static class ModbusDriverFactoryExtensions public int? TimeMs { get; init; } /// Gets or sets the keep-alive interval in milliseconds. public int? IntervalMs { get; init; } + /// Gets or sets the keep-alive time as a TimeSpan string (the AdminUI form's shape). + public JsonElement? Time { get; init; } + /// Gets or sets the keep-alive interval as a TimeSpan string (the AdminUI form's shape). + public JsonElement? Interval { get; init; } /// Gets or sets the retry count. public int? RetryCount { get; init; } } @@ -216,6 +243,10 @@ public static class ModbusDriverFactoryExtensions public int? InitialDelayMs { get; init; } /// Gets or sets the maximum reconnect delay in milliseconds. public int? MaxDelayMs { get; init; } + /// Gets or sets the initial reconnect delay as a TimeSpan string (the AdminUI form's shape). + public JsonElement? InitialDelay { get; init; } + /// Gets or sets the maximum reconnect delay as a TimeSpan string (the AdminUI form's shape). + public JsonElement? MaxDelay { get; init; } /// Gets or sets the backoff multiplier for exponential reconnect delays. public double? BackoffMultiplier { get; init; } } @@ -228,6 +259,10 @@ public static class ModbusDriverFactoryExtensions public int? IntervalMs { get; init; } /// Gets or sets the probe timeout in milliseconds. public int? TimeoutMs { get; init; } + /// Gets or sets the probe interval as a TimeSpan string (the AdminUI form's shape). + public JsonElement? Interval { get; init; } + /// Gets or sets the probe timeout as a TimeSpan string (the AdminUI form's shape). + public JsonElement? Timeout { get; init; } /// Gets or sets the probe register address. public ushort? ProbeAddress { get; init; } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuFraming.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuFraming.cs index b7f20d05..9fd9c682 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuFraming.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusRtuFraming.cs @@ -93,9 +93,10 @@ public static class ModbusRtuFraming // Exception frame: excCode(1) + CRC(2). trailing = 1 + 2; } - else if (fc is 0x01 or 0x02 or 0x03 or 0x04) + else if (fc is 0x01 or 0x02 or 0x03 or 0x04 or 0x17) { - // Read response: byteCount(1) then byteCount data bytes + CRC(2). + // Read response (and FC23 Read/Write Multiple, whose reply is a read): byteCount(1) then + // byteCount data bytes + CRC(2). var bc = new byte[1]; await ReadExactlyAsync(stream, bc, ct).ConfigureAwait(false); var byteCount = bc[0]; @@ -105,9 +106,9 @@ public static class ModbusRtuFraming } else { - // Fixed 4-byte echo: correct for the write FCs this driver emits (05/06/0F/10). A future - // variable-length response FC outside 01-04 (e.g. FC23) would be mis-sized here and - // surface as a desync — revisit the FC-shape table if the driver starts emitting one. + // Fixed 4-byte echo: correct for the write FCs this driver emits (05/06/0F/10). FC23 is + // variable-length and handled above; any NEW variable-length response FC must be added there + // or it is mis-sized here and surfaces as a desync. trailing = 4 + 2; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor index a46a255e..ce1d77fc 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor @@ -112,7 +112,7 @@
- +
diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor index a3b38dd4..86842227 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor @@ -4,24 +4,38 @@ @using ZB.MOM.WW.OtOpcUa.AdminUI.Components.Shared.Drivers.Pickers
+
+
+
+ +
+ @if (_addressError is not null) + { +
@_addressError
+ } + else if (_m.AddressString is not null) + { +
Resolved from @_m.AddressString (family @_family) — the driver re-resolves it with its own family. Editing a field below drops it.
+ }
- @foreach (var v in Enum.GetValues()) { }
-
+
- @foreach (var v in Enum.GetValues()) { }
- @foreach (var v in Enum.GetValues()) { }
-
+
-
+
@@ -66,6 +80,8 @@ if (ConfigJson == _lastConfigJson) { return; } _lastConfigJson = ConfigJson; _m = ModbusTagConfigModel.FromJson(ConfigJson); + _addressText ??= _m.AddressString; + _family = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()).Family; } private static int ParseInt(object? v, int fallback = 0) => int.TryParse(v?.ToString(), out var i) ? i : fallback; @@ -80,6 +96,25 @@ await ConfigJsonChanged.InvokeAsync(_m.ToJson()); } + private string? _addressText; + private string? _addressError; + private ModbusFamily _family = ModbusFamily.Generic; + + // A hand edit of any resolved field makes those fields authoritative: drop the address string, which the + // driver would otherwise re-resolve over them. + private Task UpdateAddress(Action apply) => Update(() => { _m.ClearAddressString(); _addressError = null; apply(); }); + + // Resolve the typed address with the OWNING driver's family (DL205 octal V-memory, MELSEC D/X/Y) and persist + // both the address string and the resolved region/offset/type fields. + private async Task ResolveAddressAsync() + { + var (family, sub) = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()); + _family = family; + var error = _m.ResolveAddress(_addressText, family, sub); + _addressError = error; + if (error is null) await ConfigJsonChanged.InvokeAsync(_m.ToJson()); + } + // Parse the builder string (e.g. "4x00001-1") back into Region + Address fields. // Format: {prefix}{offset:00000}-{length} where prefix is 0x/1x/3x/4x. private async Task OnAddressPicked(string address) @@ -94,7 +129,7 @@ var offsetStr = dashIdx >= 0 ? rest[..dashIdx] : rest; if (int.TryParse(offsetStr, out var offset)) { - await Update(() => { _m.Region = region; _m.Address = offset; }); + await UpdateAddress(() => { _m.Region = region; _m.Address = offset; }); } } } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs index 29050fc4..3ce00ad8 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs @@ -30,6 +30,74 @@ public sealed class ModbusTagConfigModel /// key stays absent. public bool? Writable { get; set; } + /// + /// The address in the parser grammar (40001:F:CDAB, DL205 V2000, MELSEC D100) when + /// the tag was authored that way; null for a tag authored by region + numeric address. When + /// present the driver re-resolves it with ITS family at runtime, so it is authoritative — editing any + /// resolved field by hand clears it (). + /// + public string? AddressString { get; set; } + + /// + /// Resolves an address string through with the owning driver's + /// family and copies the result into the region / offset / type / bit / string-length / byte-order / + /// array fields — the "persist the resolved region/offset" half of the family wiring, so the blob is + /// readable on its own and the editor shows what the driver will poll. + /// + /// The address text the operator typed. + /// The driver's PLC family. + /// The driver's MELSEC sub-family. + /// null on success, else the parser's error. + public string? ResolveAddress(string? text, ModbusFamily family, MelsecFamily melsecSubFamily) + { + if (!ModbusAddressParser.TryParse(text, family, melsecSubFamily, out var parsed, out var error) || parsed is null) + return error ?? "Address did not parse."; + AddressString = text!.Trim(); + Region = parsed.Region; + Address = parsed.Offset; + DataType = parsed.DataType; + ByteOrder = parsed.ByteOrder; + BitIndex = parsed.Bit ?? 0; + StringLength = parsed.StringLength; + if (parsed.ArrayCount is { } count) + { + TagConfigJson.Set(_bag, "isArray", true); + TagConfigJson.Set(_bag, "arrayLength", count); + } + else + { + _bag.Remove("isArray"); + _bag.Remove("arrayLength"); + } + return null; + } + + /// Drops the address string so the hand-edited region / address fields are what the driver reads. + public void ClearAddressString() => AddressString = null; + + /// + /// Reads the owning driver's family / melsecSubFamily from its DriverConfig JSON (keys + /// matched case-insensitively, like the driver's DTO binding); Generic / Q_L_iQR when absent or unknown. + /// + /// The driver's DriverConfig JSON. + /// The family pair the driver will resolve address strings with. + public static (ModbusFamily Family, MelsecFamily MelsecSubFamily) ReadFamily(string? driverConfigJson) + { + var family = ModbusFamily.Generic; + var sub = MelsecFamily.Q_L_iQR; + var o = TagConfigJson.ParseOrNew(driverConfigJson); + foreach (var (key, node) in o) + { + var text = node is JsonValue v && v.TryGetValue(out var str) ? str : null; + if (text is null) continue; + if (string.Equals(key, "family", StringComparison.OrdinalIgnoreCase) + && Enum.TryParse(text, ignoreCase: true, out var f)) family = f; + else if (string.Equals(key, "melsecSubFamily", StringComparison.OrdinalIgnoreCase) + && Enum.TryParse(text, ignoreCase: true, out var m)) sub = m; + } + return (family, sub); + } + /// /// Changes the region and keeps the data type compatible with it (the driver refuses a bit region /// with a non-Bool type, and a register region with Bool): switching to Coils / DiscreteInputs selects @@ -69,6 +137,8 @@ public sealed class ModbusTagConfigModel BitIndex = TagConfigJson.GetInt(o, "bitIndex"), StringLength = TagConfigJson.GetInt(o, "stringLength"), Writable = TagConfigJson.GetBoolNullable(o, "writable"), + AddressString = string.IsNullOrWhiteSpace(TagConfigJson.GetString(o, "addressString")) + ? null : TagConfigJson.GetString(o, "addressString"), _bag = o, }; } @@ -85,6 +155,7 @@ public sealed class ModbusTagConfigModel TagConfigJson.Set(_bag, "bitIndex", BitIndex); TagConfigJson.Set(_bag, "stringLength", StringLength); TagConfigJson.Set(_bag, "writable", Writable); + TagConfigJson.Set(_bag, "addressString", AddressString); return TagConfigJson.Serialize(_bag); } @@ -124,7 +195,7 @@ public sealed class ModbusTagConfigModel ?? TagConfigJson.DescribeInvalidEnum(_bag, "stringByteOrder") ?? TagConfigParseGate.Describe( _source, - j => ModbusTagDefinitionFactory.FromTagConfig(j, TagConfigParseGate.AuthoringRawPath, out _), + j => ModbusTagDefinitionFactory.FromTagConfigAnyFamily(j, TagConfigParseGate.AuthoringRawPath, out _), ModbusTagDefinitionFactory.Inspect, "Modbus"); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs index 857ce3b7..7aeefb70 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs @@ -86,6 +86,37 @@ public sealed class CommsRecoveryTests(ModbusSimulatorFixture sim) driver.GetHealth().State.ShouldBe(DriverState.Healthy); } + /// + /// U1 (DisableFC23 wired): a multi-register write on the default (FC23 write + read-back) path + /// round-trips against the real server — whether it implements FC23 or answers Illegal Function and + /// is moved to FC16 — and so does the forced-FC16 path. Uses the standard profile's scratch HR[200..209]. + /// + [Theory] + [InlineData(false, 200)] + [InlineData(true, 202)] + public async Task Multi_register_write_round_trips_with_and_without_FC23(bool disableFc23, int address) + { + SkipUnlessStandardLive(); + var tag = new ModbusTagDefinition("F", ModbusRegion.HoldingRegisters, (ushort)address, ModbusDataType.Float32); + var opts = new ModbusDriverOptions + { + Host = sim.Host, Port = sim.Port, UnitId = 1, Timeout = TimeSpan.FromSeconds(2), DisableFC23 = disableFc23, + RawTags = ModbusRawTags.Entries([tag]), Probe = new ModbusProbeOptions { Enabled = false }, + }; + await using var driver = new ModbusDriver(opts, "modbus-u1"); + await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); + + var value = disableFc23 ? 12.25f : 6.5f; + for (var i = 0; i < 2; i++) // second write exercises the latched path when the server lacks FC23 + { + var w = await driver.WriteAsync([new WriteRequest("F", value + i)], TestContext.Current.CancellationToken); + w[0].StatusCode.ShouldBe(StatusGood); + var r = await driver.ReadAsync(["F"], TestContext.Current.CancellationToken); + r[0].Value.ShouldBe(value + i); + } + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + /// /// H2: a deadbanded tag whose device returns holding the SAME value must publish the Good recovery. /// HR[5] is a constant 5 in the standard profile, so the recovery value is identical by construction. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDriverTests.cs index 7a686ace..457d0dfb 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusDriverTests.cs @@ -46,6 +46,7 @@ public sealed class ModbusDriverTests 0x06 => Task.FromResult(WriteSingleReg(pdu)), 0x0F => Task.FromResult(WriteMultipleCoils(pdu)), 0x10 => Task.FromResult(WriteMultipleRegs(pdu)), + 0x17 => Task.FromResult(ReadWriteMultipleRegs(pdu)), _ => Task.FromException(new ModbusException(fc, 0x01, $"fc={fc} not supported by fake")), }; } @@ -102,6 +103,26 @@ public sealed class ModbusDriverTests return new byte[] { 0x10, pdu[1], pdu[2], pdu[3], pdu[4] }; } + // FC23 Read/Write Multiple Registers: write first, then read back the read range. + private byte[] ReadWriteMultipleRegs(byte[] pdu) + { + var readAddr = (ushort)((pdu[1] << 8) | pdu[2]); + var readQty = (ushort)((pdu[3] << 8) | pdu[4]); + var writeAddr = (ushort)((pdu[5] << 8) | pdu[6]); + var writeQty = (ushort)((pdu[7] << 8) | pdu[8]); + for (var i = 0; i < writeQty; i++) + HoldingRegisters[writeAddr + i] = (ushort)((pdu[10 + i * 2] << 8) | pdu[11 + i * 2]); + var resp = new byte[2 + readQty * 2]; + resp[0] = 0x17; + resp[1] = (byte)(readQty * 2); + for (var i = 0; i < readQty; i++) + { + resp[2 + i * 2] = (byte)(HoldingRegisters[readAddr + i] >> 8); + resp[3 + i * 2] = (byte)(HoldingRegisters[readAddr + i] & 0xFF); + } + return resp; + } + private byte[] WriteMultipleCoils(byte[] pdu) { var addr = (ushort)((pdu[1] << 8) | pdu[2]); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs new file mode 100644 index 00000000..6ed8e509 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs @@ -0,0 +1,179 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// Review 07 U1: Family, MelsecSubFamily and DisableFC23 were authored in the +/// AdminUI and consumed by nothing. A tag's addressString now resolves through +/// with the driver's family; DisableFC23 gates the FC23 +/// write-and-read-back path for multi-register writes. Also pins that the AdminUI form's own +/// (TimeSpan-shaped) config keys reach the driver. +/// +[Trait("Category", "Unit")] +public sealed class ModbusFamilyWiringTests +{ + // ---------------- addressString through the family-aware parser ---------------- + + [Fact] + public void A_MELSEC_D_register_resolves_only_under_the_MELSEC_family() + { + const string json = """{"addressString":"D100"}"""; + ModbusTagDefinitionFactory.FromTagConfig(json, "T", ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); + def.Region.ShouldBe(ModbusRegion.HoldingRegisters); + def.Address.ShouldBe((ushort)100); + // Positive control: the same text means nothing to a Generic (Modicon) driver. + ModbusTagDefinitionFactory.FromTagConfig(json, "T", ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out _).ShouldBeFalse(); + } + + [Fact] + public void A_DL205_V_memory_address_resolves_octal_under_DL205() + { + ModbusTagDefinitionFactory.FromTagConfig("""{"addressString":"V2000"}""", "T", ModbusFamily.DL205, MelsecFamily.Q_L_iQR, out var def) + .ShouldBeTrue(); + def.Region.ShouldBe(ModbusRegion.HoldingRegisters); + def.Address.ShouldBe((ushort)0x400); // V2000 octal = 1024 + } + + [Fact] + public void The_address_string_carries_type_order_and_count_and_keeps_the_other_keys() + { + ModbusTagDefinitionFactory.FromTagConfig( + """{"addressString":"40011:F:CDAB:4","writable":false,"unitId":3,"region":"Coils","address":9}""", + "T", ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); + def.Region.ShouldBe(ModbusRegion.HoldingRegisters, "addressString is authoritative over stale resolved keys"); + def.Address.ShouldBe((ushort)10); + def.DataType.ShouldBe(ModbusDataType.Float32); + def.ByteOrder.ShouldBe(ModbusByteOrder.WordSwap); + def.ArrayCount.ShouldBe(4); + def.Writable.ShouldBeFalse(); + def.UnitId.ShouldBe((byte)3); + } + + [Fact] + public void An_unparseable_address_string_rejects_the_tag_and_the_deploy_gate_says_why() + { + const string json = """{"addressString":"ZZ9"}"""; + ModbusTagDefinitionFactory.FromTagConfig(json, "T", out _).ShouldBeFalse(); + ModbusTagDefinitionFactory.Inspect(json).ShouldContain(w => w.Contains("addressString")); + // The family-unaware gate accepts what SOME family resolves (it must never be stricter than the driver). + ModbusTagDefinitionFactory.Inspect("""{"addressString":"D100"}""").ShouldBeEmpty(); + } + + [Fact] + public async Task The_driver_resolves_address_strings_with_its_configured_family() + { + var fake = new ScriptedTransport(); + fake.Bank.HoldingRegisters[100] = 1234; + var opts = new ModbusDriverOptions + { + Host = "f", Family = ModbusFamily.MELSEC, Probe = new ModbusProbeOptions { Enabled = false }, + RawTags = [new RawTagEntry("D100", """{"addressString":"D100"}""", false)], + }; + await using var drv = new ModbusDriver(opts, "m1", _ => fake); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + var r = await drv.ReadAsync(["D100"], TestContext.Current.CancellationToken); + r[0].StatusCode.ShouldBe(0u); + r[0].Value.ShouldBe((short)1234); + } + + [Fact] + public void Family_and_MelsecSubFamily_parse_from_the_driver_config() + { + var o = ModbusDriverFactoryExtensions.ParseOptions("m1", """{"host":"h","family":"MELSEC","melsecSubFamily":"F_iQF","disableFC23":true}"""); + o.Family.ShouldBe(ModbusFamily.MELSEC); + o.MelsecSubFamily.ShouldBe(MelsecFamily.F_iQF); + o.DisableFC23.ShouldBeTrue(); + } + + // ---------------- DisableFC23 gates the FC23 multi-register write ---------------- + + private static async Task<(ModbusDriver, ScriptedTransport)> StartWriteAsync(bool disableFc23) + { + var t = new ScriptedTransport(); + var opts = new ModbusDriverOptions + { + Host = "f", DisableFC23 = disableFc23, Probe = new ModbusProbeOptions { Enabled = false }, + RawTags = ModbusRawTags.Entries([new ModbusTagDefinition("F", ModbusRegion.HoldingRegisters, 20, ModbusDataType.Float32)]), + }; + var drv = new ModbusDriver(opts, "m1", _ => t); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + return (drv, t); + } + + [Fact] + public async Task Multi_register_writes_use_FC23_write_and_read_back_by_default() + { + var (drv, t) = await StartWriteAsync(disableFc23: false); + await using var _ = drv; + var w = await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken); + w[0].StatusCode.ShouldBe(0u); + t.Sent.Select(s => s.Pdu[0]).ShouldBe([(byte)0x17]); + (await drv.ReadAsync(["F"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(1.5f); + } + + [Fact] + public async Task DisableFC23_writes_with_FC16() + { + var (drv, t) = await StartWriteAsync(disableFc23: true); + await using var _ = drv; + (await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); + t.Sent.Select(s => s.Pdu[0]).ShouldBe([(byte)0x10]); + } + + [Fact] + public async Task A_device_without_FC23_falls_back_to_FC16_once_and_stays_on_it() + { + var (drv, t) = await StartWriteAsync(disableFc23: false); + await using var _ = drv; + t.Handler = (_, pdu, _) => pdu[0] == 0x17 + ? Task.FromException(new ModbusException(0x17, 0x01, "illegal function")) + : null; + + (await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); + (await drv.WriteAsync([new WriteRequest("F", 2.5f)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); + + t.Sent.Select(s => s.Pdu[0]).ShouldBe([(byte)0x17, (byte)0x10, (byte)0x10]); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + // ---------------- the AdminUI form's own keys reach the driver ---------------- + + [Fact] + public void The_AdminUI_form_shape_of_the_timing_keys_is_honoured() + { + // The form serialises ModbusDriverOptions itself (camelCase, TimeSpan strings), not the *Ms DTO keys. + var formShape = JsonSerializer.Serialize(new ModbusDriverOptions + { + Host = "h", + Timeout = TimeSpan.FromSeconds(7), + IdleDisconnectTimeout = TimeSpan.FromSeconds(90), + AutoProhibitReprobeInterval = TimeSpan.FromSeconds(45), + Probe = new ModbusProbeOptions { Interval = TimeSpan.FromSeconds(11), Timeout = TimeSpan.FromSeconds(3) }, + KeepAlive = new ModbusKeepAliveOptions { Time = TimeSpan.FromSeconds(60), Interval = TimeSpan.FromSeconds(20) }, + Reconnect = new ModbusReconnectOptions { InitialDelay = TimeSpan.FromSeconds(2), MaxDelay = TimeSpan.FromSeconds(40) }, + }, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, Converters = { new JsonStringEnumConverter() } }); + + var o = ModbusDriverFactoryExtensions.ParseOptions("m1", formShape); + + o.Timeout.ShouldBe(TimeSpan.FromSeconds(7)); + o.IdleDisconnectTimeout.ShouldBe(TimeSpan.FromSeconds(90)); + o.AutoProhibitReprobeInterval.ShouldBe(TimeSpan.FromSeconds(45)); + o.Probe.Interval.ShouldBe(TimeSpan.FromSeconds(11)); + o.Probe.Timeout.ShouldBe(TimeSpan.FromSeconds(3)); + o.KeepAlive.Time.ShouldBe(TimeSpan.FromSeconds(60)); + o.KeepAlive.Interval.ShouldBe(TimeSpan.FromSeconds(20)); + o.Reconnect.InitialDelay.ShouldBe(TimeSpan.FromSeconds(2)); + o.Reconnect.MaxDelay.ShouldBe(TimeSpan.FromSeconds(40)); + } + + [Fact] + public void The_millisecond_keys_still_win_when_both_are_present() + { + var o = ModbusDriverFactoryExtensions.ParseOptions("m1", """{"host":"h","timeoutMs":1500,"timeout":"00:00:07"}"""); + o.Timeout.ShouldBe(TimeSpan.FromMilliseconds(1500)); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRtuFramingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRtuFramingTests.cs index 3a26fa28..6c084904 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRtuFramingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRtuFramingTests.cs @@ -25,6 +25,17 @@ public sealed class ModbusRtuFramingTests pdu.ShouldBe(new byte[] { 0x03, 0x02, 0x00, 0x0A }); // fc + byteCount + data, no addr/CRC } + /// FC23 (Read/Write Multiple) replies like a read: byte count + data. It used to be sized as a + /// fixed 4-byte write echo and desync. + [Fact] + public async Task ReadResponse_FC23_returns_bare_pdu() + { + var body = new byte[] { 0x01, 0x17, 0x04, 0x00, 0x0A, 0x00, 0x0B }; + await using var s = Canned(ModbusCrc.Append(body)); + var pdu = await ModbusRtuFraming.ReadResponsePduAsync(s, 0x01, TestContext.Current.CancellationToken); + pdu.ShouldBe(new byte[] { 0x17, 0x04, 0x00, 0x0A, 0x00, 0x0B }); + } + [Fact] public async Task ReadResponse_exception_frame_throws_ModbusException() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs index e1afd0c1..49abe9ab 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs @@ -141,4 +141,43 @@ public sealed class ModbusTagConfigModelTests [Fact] public void Validate_refuses_an_incompatible_region_and_type() => ModbusTagConfigModel.FromJson("""{"region":"Coils","address":1,"dataType":"Int16"}""").Validate().ShouldNotBeNull(); + + /// Review 07 U1: the editor resolves an address string with the DRIVER's family and persists + /// both the string and the resolved fields; the driver re-resolves the string with its family. + [Fact] + public void ResolveAddress_uses_the_driver_family_and_persists_string_and_resolved_fields() + { + var (family, sub) = ModbusTagConfigModel.ReadFamily("""{"Family":"MELSEC","melsecSubFamily":"Q_L_iQR"}"""); + family.ShouldBe(ModbusFamily.MELSEC); + + var m = ModbusTagConfigModel.FromJson("{}"); + m.ResolveAddress("D100", family, sub).ShouldBeNull(); + m.Region.ShouldBe(ModbusRegion.HoldingRegisters); + m.Address.ShouldBe(100); + + var json = m.ToJson(); + json.ShouldContain("\"addressString\":\"D100\""); + json.ShouldContain("\"address\":100"); + ModbusTagDefinitionFactory.FromTagConfig(json, "T", ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); + def.Address.ShouldBe((ushort)100); + ModbusTagConfigModel.FromJson(json).Validate().ShouldBeNull("the family-unaware authoring gate accepts what the MELSEC driver serves"); + + // Positive control: under Generic the same text is an error, and nothing is persisted. + var g = ModbusTagConfigModel.FromJson("{}"); + g.ResolveAddress("D100", ModbusFamily.Generic, MelsecFamily.Q_L_iQR).ShouldNotBeNull(); + g.AddressString.ShouldBeNull(); + } + + [Fact] + public void A_hand_edit_after_resolving_drops_the_address_string() + { + var m = ModbusTagConfigModel.FromJson("{}"); + m.ResolveAddress("40011:F", ModbusFamily.Generic, MelsecFamily.Q_L_iQR).ShouldBeNull(); + m.ClearAddressString(); + m.Address = 50; + var json = m.ToJson(); + json.ShouldNotContain("addressString"); + ModbusTagDefinitionFactory.FromTagConfig(json, "T", out var def).ShouldBeTrue(); + def.Address.ShouldBe((ushort)50); + } } From 6ffd881d425435d1be9450c703d877f530fe1bba Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:37:41 -0400 Subject: [PATCH 13/34] fix(modbus): a failed read or write drops the WriteOnChangeOnly suppression for the tag After a comms fault the device value is unknown, so re-asserting the same setpoint must reach the wire - the write-side twin of the value-only recovery dedup trap. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriver.cs | 17 ++++++++++++ .../ModbusHealthContractTests.cs | 26 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index 2bc8e8ef..ad9b7295 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -746,6 +746,7 @@ public sealed class ModbusDriver // Fail fast: this unit (or the whole endpoint) already failed at the transport this batch; // paying the same timeout once per remaining tag would stall the poll N-fold. results[i] = new DataValueSnapshot(null, StatusBadCommunicationError, null, now); + InvalidateWriteCache(fullReferences[i]); continue; } try @@ -767,6 +768,7 @@ public sealed class ModbusDriver catch (Exception ex) { results[i] = new DataValueSnapshot(null, ClassifyFailure(ex, unit, fullReferences[i], batch, isWrite: false), null, now); + InvalidateWriteCache(fullReferences[i]); } } @@ -1253,6 +1255,7 @@ public sealed class ModbusDriver foreach (var (idx, _) in block.Members) { results[idx] = new DataValueSnapshot(null, StatusBadCommunicationError, null, timestamp); + InvalidateWriteCache(fullReferences[idx]); handled.Add(idx); } continue; @@ -1285,6 +1288,7 @@ public sealed class ModbusDriver foreach (var (idx, _) in block.Members) { results[idx] = new DataValueSnapshot(null, status, null, timestamp); + InvalidateWriteCache(fullReferences[idx]); handled.Add(idx); } continue; @@ -1341,6 +1345,17 @@ public sealed class ModbusDriver ? ReadBitBlockAsync(transport, unit, ReadFunctionCode(region), start, qty, ct) : ReadRegisterBlockAsync(transport, unit, ReadFunctionCode(region), start, qty, ct); + /// + /// A failed read leaves the device's value UNKNOWN (it may have rebooted holding anything), so the + /// WriteOnChangeOnly suppression for that tag is dropped — otherwise re-asserting the same setpoint + /// after an outage would be swallowed, the write-side twin of the value-only recovery dedup trap. + /// + private void InvalidateWriteCache(string fullRef) + { + if (!_options.WriteOnChangeOnly) return; + lock (_lastWrittenLock) _lastWrittenByRef.Remove(fullRef); + } + private void InvalidateWriteCacheIfDiverged(string fullRef, object value) { if (!_options.WriteOnChangeOnly) return; @@ -1483,6 +1498,8 @@ public sealed class ModbusDriver catch (Exception ex) { results[i] = new WriteResult(MapWriteFailure(ex, unit, w.FullReference, batch, cancellationToken)); + // A failed write may or may not have landed — the device value is unknown now. + InvalidateWriteCache(w.FullReference); } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs index 50282ec3..0e591189 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs @@ -215,6 +215,32 @@ public sealed class ModbusHealthContractTests drv.GetHealth().State.ShouldBe(DriverState.Healthy); } + [Fact] + public async Task A_failed_read_invalidates_the_WriteOnChangeOnly_cache() + { + // Value-only suppression is the write-side twin of the #527 dedup trap: after a comms fault the + // device's register is unknown (it may have rebooted), so re-asserting the same setpoint must reach it. + var opts = new ModbusDriverOptions + { + Host = "f", RawTags = ModbusRawTags.Entries([Hr("A", 10)]), WriteOnChangeOnly = true, + Probe = new ModbusProbeOptions { Enabled = false }, + }; + var (drv, t) = await StartAsync(opts); + await using var _ = drv; + int Fc06() => t.Sent.Count(s => s.Pdu[0] == 0x06); + + await drv.WriteAsync([new WriteRequest("A", (short)5)], TestContext.Current.CancellationToken); + await drv.WriteAsync([new WriteRequest("A", (short)5)], TestContext.Current.CancellationToken); + Fc06().ShouldBe(1, "positive control: an identical re-write is suppressed while nothing failed"); + + t.Handler = (_, pdu, _) => pdu[0] == 0x03 ? Task.FromException(new IOException("dropped")) : null; + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); + t.Handler = null; + + await drv.WriteAsync([new WriteRequest("A", (short)5)], TestContext.Current.CancellationToken); + Fc06().ShouldBe(2, "after a failed read the device value is unknown; the write must go to the wire"); + } + // ---------------- lifecycle generation ---------------- [Fact] From 6a51025012522971f469ef4c08ffac4932aba94e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:38:29 -0400 Subject: [PATCH 14/34] perf(mqtt): wildcard topic trie and a shared strict UTF-8 decoder (R06:M-6, M-7) Wildcard-authored tags were matched by one MqttTopicFilterComparer call per tag per inbound message on the dispatcher thread. They now live in an allocation-free level trie whose candidates are confirmed by the same comparer (semantics unchanged; pinned differentially against the comparer). TryDecodeUtf8 reuses one static throw-on-invalid UTF8Encoding instead of allocating one per Raw/Scalar delivery. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../MqttSubscriptionManager.cs | 76 +++++++-- .../TopicFilterTrie.cs | 152 ++++++++++++++++++ .../TopicFilterTrieTests.cs | 96 +++++++++++ 3 files changed, 311 insertions(+), 13 deletions(-) create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 1504328a..0c8b1a9a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -48,7 +48,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt; /// accepted by the parser (warned at deploy, see MqttTagDefinitionFactory.Inspect), so /// matching uses MQTTnet's own rather than string /// equality — the same comparer a broker applies. Concrete topics stay a single dictionary -/// lookup; the linear wildcard scan runs only when a wildcard tag actually exists. +/// lookup; wildcard filters live in a level trie (TopicFilterTrie) whose candidates the comparer +/// confirms, so a thousand wildcard tags cost a walk over the topic's levels, not a thousand compares. /// /// /// Sparkplug seam. P1 ingests plain topics only. Sparkplug B decodes a @@ -74,6 +75,13 @@ public sealed class MqttSubscriptionManager /// public const int DefaultQos = 1; + /// + /// Throw-on-invalid UTF-8, shared (R06:M-7): it is stateless and thread-safe, and allocating one per + /// Raw/Scalar delivery was per-message garbage on the dispatcher thread. Same choice + /// MqttBrowseSession makes. + /// + private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + // OPC UA status codes (values verified against Opc.Ua.StatusCodes, not recalled). private const uint StatusGood = 0x00000000u; @@ -644,13 +652,14 @@ public sealed class MqttSubscriptionManager } // A published topic name may not itself contain a wildcard, so a wildcard-authored - // definition can never also be an exact-index hit: no tag is delivered twice. - foreach (var def in authored.Wildcards) + // definition can never also be an exact-index hit: no tag is delivered twice. The trie walks + // the topic's levels (R06:M-6 — this used to be one comparer call per wildcard tag, per + // message, on the shared dispatcher thread) and every candidate it yields is confirmed by + // MQTTnet's own comparer inside the visitor, so the semantics are the broker's. + if (authored.WildcardTrie.Count > 0) { - if (MqttTopicFilterComparer.Compare(topic, def.Topic) == MqttTopicFilterCompareResult.IsMatch) - { - Deliver(def, payload, retained, now, oversized, ref json); - } + var visitor = new WildcardDelivery(this, topic, payload, retained, now, oversized, ref json); + authored.WildcardTrie.Match(topic, ref visitor); } } finally @@ -1447,8 +1456,7 @@ public sealed class MqttSubscriptionManager { try { - text = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) - .GetString(payload); + text = Utf8Strict.GetString(payload); return true; } catch (DecoderFallbackException) @@ -1476,6 +1484,47 @@ public sealed class MqttSubscriptionManager return $"mqtt:{_mode}:{id}:[{named}{overflow}]"; } + /// + /// Delivers each wildcard-trie candidate whose filter the MQTTnet comparer confirms. A ref struct + /// so the payload span and the fan-out's shared JSON parse ride along without a closure or a list. + /// + private ref struct WildcardDelivery : ITopicFilterVisitor + { + private readonly MqttSubscriptionManager _manager; + private readonly string _topic; + private readonly ReadOnlySpan _payload; + private readonly bool _retained; + private readonly DateTime _now; + private readonly bool _oversized; + private ref PayloadJson _json; + + public WildcardDelivery( + MqttSubscriptionManager manager, + string topic, + ReadOnlySpan payload, + bool retained, + DateTime now, + bool oversized, + ref PayloadJson json) + { + _manager = manager; + _topic = topic; + _payload = payload; + _retained = retained; + _now = now; + _oversized = oversized; + _json = ref json; + } + + public void Visit(MqttTagDefinition def) + { + if (MqttTopicFilterComparer.Compare(_topic, def.Topic) == MqttTopicFilterCompareResult.IsMatch) + { + _manager.Deliver(def, _payload, _retained, _now, _oversized, ref _json); + } + } + } + /// Opaque subscription identity; the diagnostic id names the mode and its topics. /// The human-readable identity surfaced for diagnostics. private sealed record MqttSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle; @@ -1487,7 +1536,8 @@ public sealed class MqttSubscriptionManager /// /// Two topic indexes, and the difference is load-bearing. Delivery matches an /// incoming published topic, which never carries a wildcard, so it uses - /// (concrete only) plus a comparer scan of . + /// (concrete only) plus a walk whose candidates the + /// comparer confirms. /// Every path that instead reconstructs state from a subscribed filter string — /// reconnect re-subscribe and per-topic degradation — must use , which is /// keyed by the tag's authored topic including wildcard patterns, because that is what @@ -1501,12 +1551,12 @@ public sealed class MqttSubscriptionManager /// included. Keyed the same way _subscribedTopics and SUBACK outcomes are. /// /// Concrete topic → its definitions; the delivery fast path. - /// Definitions whose authored topic carries an MQTT wildcard. + /// Definitions whose authored topic carries an MQTT wildcard, by filter level. private sealed record AuthoredTable( FrozenDictionary ByRawPath, FrozenDictionary ByFilter, FrozenDictionary ByExactTopic, - MqttTagDefinition[] Wildcards) + TopicFilterTrie WildcardTrie) { public static readonly AuthoredTable Empty = Build(new Dictionary(StringComparer.Ordinal)); @@ -1556,7 +1606,7 @@ public sealed class MqttSubscriptionManager byRawPath.ToFrozenDictionary(StringComparer.Ordinal), byFilter.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.ToArray(), StringComparer.Ordinal), concrete.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.ToArray(), StringComparer.Ordinal), - [.. wildcards]); + TopicFilterTrie.Build(wildcards.Select(d => (d.Topic, d)))); } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs new file mode 100644 index 00000000..976523f7 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs @@ -0,0 +1,152 @@ +using System.Collections.Frozen; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt; + +/// Receives each candidate a match produces. +/// The value type stored against each filter. +internal interface ITopicFilterVisitor +{ + /// Called once per value whose filter may match the topic. + /// The candidate. + void Visit(T value); +} + +/// +/// An immutable trie over MQTT topic FILTERS (+ / # wildcards), so matching one published +/// topic costs a walk over its levels instead of a comparer call per wildcard-authored tag (R06:M-6: +/// the linear scan ran on MQTTnet's shared dispatcher thread for every inbound message). +/// +/// +/// +/// A candidate generator, not the authority. The trie implements the plain MQTT 3.1.1 §4.7 +/// level rules — + is exactly one level (an empty one included), # is the parent level +/// and everything under it — and deliberately nothing subtler (the $-topic exclusion, shared +/// subscriptions). It can therefore only ever return a SUPERSET of the true matches, and every +/// caller confirms each candidate with MQTTnet's own MqttTopicFilterComparer, the comparer a +/// broker applies. Semantics stay byte-identical to the scan it replaces; only the work changes. +/// A differential test pins both halves against that comparer. +/// +/// +/// Allocation-free per match. Levels are walked as spans and looked up through the +/// dictionary's span alternate lookup; candidates are pushed to a caller-supplied visitor (a +/// ref struct in production) rather than collected into a list. +/// +/// +/// The value type stored against each filter. +internal sealed class TopicFilterTrie +{ + private readonly Node _root; + + private TopicFilterTrie(Node root, int count) + { + _root = root; + Count = count; + } + + /// An empty trie. + public static TopicFilterTrie Empty { get; } = Build([]); + + /// How many values were inserted. + public int Count { get; } + + /// Builds a trie from (filter, value) pairs. + /// The filters and their values. + /// The immutable trie. + public static TopicFilterTrie Build(IEnumerable<(string Filter, T Value)> entries) + { + var root = new Builder(); + var count = 0; + foreach (var (filter, value) in entries) + { + var node = root; + var levels = filter.Split('/'); + var terminated = false; + for (var i = 0; i < levels.Length; i++) + { + var level = levels[i]; + if (level == "#") + { + // '#' must be the last level; anything after it is a malformed filter the comparer + // will refuse anyway, so treating '#' as terminal here can only over-generate. + node.Hash.Add(value); + terminated = true; + break; + } + + node = level == "+" + ? node.Plus ??= new Builder() + : node.Children.TryGetValue(level, out var child) ? child : node.Children[level] = new Builder(); + } + + if (!terminated) node.Exact.Add(value); + count++; + } + + return new TopicFilterTrie(root.Freeze(), count); + } + + /// Pushes every value whose filter may match to . + /// The visitor; a ref struct is allowed. + /// A concrete published topic (never a filter). + /// Receives each candidate — a superset of the true matches; confirm each one. + public void Match(string topic, ref TVisitor visitor) + where TVisitor : ITopicFilterVisitor, allows ref struct + { + if (Count == 0 || topic is null) return; + Walk(_root, topic.AsSpan(), exhausted: false, ref visitor); + } + + private static void Walk(Node node, ReadOnlySpan rest, bool exhausted, ref TVisitor visitor) + where TVisitor : ITopicFilterVisitor, allows ref struct + { + // '#' here covers this level's parent-match AND every deeper level. + foreach (var value in node.Hash) visitor.Visit(value); + + if (exhausted) + { + foreach (var value in node.Exact) visitor.Visit(value); + return; + } + + var slash = rest.IndexOf('/'); + var level = slash < 0 ? rest : rest[..slash]; + var next = slash < 0 ? ReadOnlySpan.Empty : rest[(slash + 1)..]; + var nextExhausted = slash < 0; + + if (node.Children.Count > 0 + && node.Children.GetAlternateLookup>().TryGetValue(level, out var child)) + { + Walk(child, next, nextExhausted, ref visitor); + } + + if (node.Plus is { } plus) Walk(plus, next, nextExhausted, ref visitor); + } + + private sealed class Node(FrozenDictionary children, Node? plus, T[] exact, T[] hash) + { + public FrozenDictionary Children { get; } = children; + + public Node? Plus { get; } = plus; + + public T[] Exact { get; } = exact; + + public T[] Hash { get; } = hash; + } + + private sealed class Builder + { + public Dictionary Children { get; } = new(StringComparer.Ordinal); + + public Builder? Plus { get; set; } + + public List Exact { get; } = []; + + public List Hash { get; } = []; + + public Node Freeze() => new( + Children.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.Freeze(), StringComparer.Ordinal), + Plus?.Freeze(), + [.. Exact], + [.. Hash]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs new file mode 100644 index 00000000..f21677e1 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs @@ -0,0 +1,96 @@ +using MQTTnet; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; + +/// +/// R06:M-6. The wildcard trie replaced a linear MqttTopicFilterComparer scan on the +/// dispatcher thread. Its contract is "a superset of the true matches, confirmed by the comparer", so it +/// is pinned differentially against that comparer — the oracle a broker uses — over a seeded random +/// space that deliberately includes empty levels, $-topics and wildcard-only filters. +/// +public sealed class TopicFilterTrieTests +{ + private static readonly string[] Levels = ["a", "b", "c", "", "$SYS", "x y"]; + + private sealed class Collect : ITopicFilterVisitor + { + public HashSet Seen { get; } = []; + + public void Visit(int value) => Seen.Add(value).ShouldBeTrue("a filter must be visited at most once per match"); + } + + [Fact] + public void Match_IsASupersetOfTheComparer_AndConfirmingYieldsExactlyTheComparersAnswer() + { + var random = new Random(744); + for (var round = 0; round < 200; round++) + { + var filters = Enumerable.Range(0, random.Next(1, 25)).Select(_ => RandomFilter(random)).ToList(); + var trie = TopicFilterTrie.Build(filters.Select((f, i) => (f, i))); + + for (var t = 0; t < 50; t++) + { + var topic = RandomTopic(random); + var expected = filters + .Select((f, i) => (f, i)) + .Where(p => MqttTopicFilterComparer.Compare(topic, p.f) == MqttTopicFilterCompareResult.IsMatch) + .Select(p => p.i) + .ToHashSet(); + + var collect = new Collect(); + trie.Match(topic, ref collect); + + collect.Seen.IsSupersetOf(expected).ShouldBeTrue($"topic '{topic}' missed a match; filters: {string.Join(" | ", filters)}"); + collect.Seen + .Where(i => MqttTopicFilterComparer.Compare(topic, filters[i]) == MqttTopicFilterCompareResult.IsMatch) + .ToHashSet() + .SetEquals(expected) + .ShouldBeTrue(); + } + } + } + + [Theory] + [InlineData("a/#", "a", true)] // '#' includes the parent level + [InlineData("a/#", "a/b/c", true)] + [InlineData("a/+", "a/", true)] // '+' matches an empty level + [InlineData("a/+", "a", false)] + [InlineData("+/+", "a/b", true)] + [InlineData("#", "anything/at/all", true)] + [InlineData("a/b", "a/b", true)] + [InlineData("a/b", "a/c", false)] + public void Match_FindsTheCanonicalCases(string filter, string topic, bool expected) + { + var trie = TopicFilterTrie.Build([(filter, 1)]); + var collect = new Collect(); + trie.Match(topic, ref collect); + collect.Seen.Contains(1).ShouldBe(expected); + } + + [Fact] + public void Empty_MatchesNothing() + { + var collect = new Collect(); + TopicFilterTrie.Empty.Match("a/b", ref collect); + collect.Seen.ShouldBeEmpty(); + } + + private static string RandomFilter(Random random) + { + var count = random.Next(1, 5); + var parts = new List(count); + for (var i = 0; i < count; i++) + { + var roll = random.Next(10); + if (roll < 2 && i == count - 1) { parts.Add("#"); break; } + parts.Add(roll < 5 ? "+" : Levels[random.Next(Levels.Length)]); + } + + return string.Join('/', parts); + } + + private static string RandomTopic(Random random) => + string.Join('/', Enumerable.Range(0, random.Next(1, 6)).Select(_ => Levels[random.Next(Levels.Length)])); +} From c529948745e0193ebcafc03c374c10b012ea9262 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:42:50 -0400 Subject: [PATCH 15/34] perf(mqtt): publish a write batch concurrently, serialised per topic; writes feed health (R06:M-8) WriteAsync published one awaited QoS 1 round-trip per request, so 50 writes to a stalled broker cost 50 publish deadlines. Different topics now publish concurrently (at most 8 in flight); same-topic writes stay in request order. A caller cancel still fills every slot. A successful publish advances LastSuccessfulRead. MiniBroker now frames packets properly and records/PUBACKs PUBLISHes. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 5 + .../MqttDriver.cs | 104 ++++++++++++--- .../MiniBroker.cs | 119 +++++++++++++++--- .../MqttDriverSessionLifecycleTests.cs | 73 +++++++++++ 4 files changed, 272 insertions(+), 29 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index b9e62582..d5c33835 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -491,6 +491,11 @@ with their real type; a birth type the encoder cannot represent (any `*Array`, D matters and is deliberately not configurable: a retained command would be replayed by the broker to the device on *every* future subscribe — including the device's own reconnect — so one stale write would be re-applied forever. +**A batch publishes concurrently, one topic at a time (R06:M-8).** Writes to *different* topics in one +`WriteAsync` go out together, at most 8 in flight, so a stalled broker costs a batch about one publish deadline +rather than one per request; writes to the *same* topic stay sequential in request order, so the last one wins. +A successful publish (PUBACK, or a sent QoS 0 command) advances `LastSuccessfulRead` like an inbound message. + **Plain QoS 1 is at-least-once.** Know the consequences before pointing a Plain `writeTopic` at a non-idempotent device topic: diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs index 26cdeab6..dec7569c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs @@ -104,6 +104,12 @@ public sealed class MqttDriver /// private const string DiscoveryFolderName = "Mqtt"; + /// + /// Ceiling on PUBLISHes one write batch keeps in flight (R06:M-8). Bounded so a large batch cannot + /// flood a broker's inflight window; different topics share it, one topic's writes stay sequential. + /// + internal const int MaxConcurrentPublishes = 8; + /// How often the host-connectivity poll samples . private static readonly TimeSpan HostProbeInterval = TimeSpan.FromSeconds(1); @@ -209,6 +215,9 @@ public sealed class MqttDriver private DriverHealth _health = new(DriverState.Unknown, null, null); + /// UTC ticks of the last successful write PUBLISH; 0 = none. Answer evidence for health. + private long _lastPublishUtcTicks; + /// Guards the generation bump against the generation-checked health write. private readonly object _healthLock = new(); @@ -446,7 +455,7 @@ public sealed class MqttDriver return stored; } - var lastMessage = connection.LastMessageUtc ?? stored.LastSuccessfulRead; + var lastMessage = Latest(connection.LastMessageUtc ?? stored.LastSuccessfulRead, LastPublishUtc); return connection.State switch { MqttConnectionState.Connected => new DriverHealth(DriverState.Healthy, lastMessage, stored.LastError), @@ -708,9 +717,12 @@ public sealed class MqttDriver { ArgumentNullException.ThrowIfNull(writes); - var results = new WriteResult[writes.Count]; + var results = new WriteResult?[writes.Count]; var connection = _connection; + // Pass 1 — everything that needs no broker: resolve, encode, refuse. What survives is grouped by + // destination topic in REQUEST order. + var byTopic = new Dictionary>(StringComparer.Ordinal); for (var i = 0; i < writes.Count; i++) { var w = writes[i]; @@ -738,22 +750,70 @@ public sealed class MqttDriver continue; } + if (!byTopic.TryGetValue(encoded.Topic, out var queue)) + { + byTopic[encoded.Topic] = queue = []; + } + + queue.Add((i, encoded, w.FullReference)); + } + + // Pass 2 — publish (R06:M-8). Different topics go out concurrently, at most MaxConcurrentPublishes + // in flight, so a stalled broker costs a batch ~one publish deadline instead of one per request. + // Writes to the SAME topic stay strictly sequential in request order: the device applies them in + // arrival order, and the last one must win. + if (byTopic.Count > 0) + { + using var slots = new SemaphoreSlim(MaxConcurrentPublishes, MaxConcurrentPublishes); + await Task.WhenAll(byTopic.Values.Select(queue => PublishQueueAsync(connection!, queue, slots, results, cancellationToken))) + .ConfigureAwait(false); + } + + // Every slot is filled — a caller cancellation leaves the unattempted ones BadCommunicationError + // rather than throwing, so the batch shape holds. + var filled = new WriteResult[writes.Count]; + for (var i = 0; i < filled.Length; i++) + { + filled[i] = results[i] ?? new WriteResult(StatusBadCommunicationError); + } + + return filled; + } + + /// Publishes one topic's writes in request order, each under a shared concurrency slot. + private async Task PublishQueueAsync( + MqttConnection connection, + List<(int Index, MqttEncodedWrite Encoded, string Reference)> queue, + SemaphoreSlim slots, + WriteResult?[] results, + CancellationToken cancellationToken) + { + foreach (var (index, encoded, reference) in queue) + { + try + { + await slots.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; // caller-cancelled: the remaining slots are filled by the caller + } + try { await connection .PublishAsync(encoded.Topic, encoded.Payload, encoded.Qos, retain: false, cancellationToken) .ConfigureAwait(false); - results[i] = new WriteResult(0u); + results[index] = new WriteResult(0u); + + // A PUBACK (or a sent QoS 0 command) is the broker answering — evidence the session works, + // so it feeds LastSuccessfulRead alongside inbound messages. + Interlocked.Exchange(ref _lastPublishUtcTicks, DateTime.UtcNow.Ticks); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - // Caller-cancelled: fill the remainder rather than throwing, so the batch shape holds. - for (var j = i; j < writes.Count; j++) - { - results[j] = new WriteResult(StatusBadCommunicationError); - } - - return results; + results[index] = new WriteResult(StatusBadCommunicationError); + return; } catch (Exception ex) { @@ -761,13 +821,15 @@ public sealed class MqttDriver ex, "MQTT driver '{DriverId}': write to '{Reference}' could not be published on '{Topic}'.", _driverInstanceId, - w.FullReference, + reference, encoded.Topic); - results[i] = new WriteResult(StatusBadCommunicationError); + results[index] = new WriteResult(StatusBadCommunicationError); + } + finally + { + slots.Release(); } } - - return results; } /// Maps an encode refusal to the status an OPC UA client sees. @@ -1528,6 +1590,20 @@ public sealed class MqttDriver OnHostStatusChanged?.Invoke(this, new HostStatusChangedEventArgs(HostName, previous, next)); } + /// The last successful write publish, or null. + private DateTime? LastPublishUtc + { + get + { + var ticks = Interlocked.Read(ref _lastPublishUtcTicks); + return ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc); + } + } + + /// The later of two optional instants. + private static DateTime? Latest(DateTime? a, DateTime? b) + => a is null ? b : b is null ? a : (a > b ? a : b); + /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs index 77d0b762..5cb47e49 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs @@ -5,7 +5,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; /// /// The smallest broker that can complete an MQTT 3.1.1 handshake: accept TCP, answer CONNECT with -/// a CONNACK, answer PINGREQ with PINGRESP, and nothing else. That is enough for MQTTnet to report +/// a CONNACK, answer PINGREQ with PINGRESP, SUBSCRIBE with a SUBACK (unless told not to), and record +/// PUBLISHes (PUBACKing QoS 1 only when told to). It never forwards a PUBLISH to a subscriber. That is enough for MQTTnet to report /// a real connection — which is what makes a real drop () drive a real /// DisconnectedAsync and a real reconnect, rather than a test seam standing in for one. /// @@ -107,6 +108,96 @@ internal sealed class MiniBroker : IDisposable return [.. ack]; } + /// + /// When true, a QoS 1 PUBLISH is answered with a PUBACK. False (the default) is a broker that takes + /// the PUBLISH and never acknowledges it — the stalled-broker shape a write deadline exists for. + /// + public bool AckPublish { get; set; } + + /// Every PUBLISH received, in arrival order, as (topic, payload). + public System.Collections.Concurrent.ConcurrentQueue<(string Topic, byte[] Payload)> Published { get; } = new(); + + /// Handles one complete packet; returns false when the connection must close. + private async Task HandlePacketAsync(NetworkStream stream, byte[] packet) + { + var packetType = packet[0] & 0xF0; + if (packetType == 0x10) + { + await stream.WriteAsync(new byte[] { 0x20, 0x02, 0x00, ConnAckReturnCode }, _cts.Token); + + // A real broker closes the socket straight after refusing, which is what drives + // MQTTnet's DisconnectedAsync. Mirroring it keeps the fake honest. + return ConnAckReturnCode == ReturnCodeAccepted; + } + + if (packetType == 0xC0) + { + await stream.WriteAsync(PingResp, _cts.Token); + } + else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(packet, packet.Length) is { } subAck) + { + await stream.WriteAsync(subAck, _cts.Token); + } + else if (packetType == 0x30) + { + var qos = (packet[0] >> 1) & 0x03; + var at = HeaderLength(packet); + var topicLength = (packet[at] << 8) | packet[at + 1]; + var topic = System.Text.Encoding.UTF8.GetString(packet, at + 2, topicLength); + at += 2 + topicLength; + byte idHigh = 0, idLow = 0; + if (qos > 0) + { + idHigh = packet[at]; + idLow = packet[at + 1]; + at += 2; + } + + Published.Enqueue((topic, packet[at..])); + if (qos == 1 && AckPublish) + { + await stream.WriteAsync(new byte[] { 0x40, 0x02, idHigh, idLow }, _cts.Token); + } + } + + return true; + } + + /// Length of a packet's fixed header (type byte + variable-length remaining length). + private static int HeaderLength(byte[] packet) + { + var at = 1; + while ((packet[at] & 0x80) != 0) at++; + return at + 1; + } + + /// Removes one complete packet from the front of , if one is there. + private static bool TryTakePacket(List pending, out byte[] packet) + { + packet = []; + if (pending.Count < 2) return false; + + var at = 1; + var remaining = 0; + var multiplier = 1; + byte digit; + do + { + if (at >= pending.Count) return false; + digit = pending[at++]; + remaining += (digit & 0x7F) * multiplier; + multiplier *= 128; + } + while ((digit & 0x80) != 0); + + var total = at + remaining; + if (pending.Count < total) return false; + + packet = pending.GetRange(0, total).ToArray(); + pending.RemoveRange(0, total); + return true; + } + /// Total TCP connections accepted since construction — counts reconnect attempts. public int AcceptedCount => Volatile.Read(ref _acceptedCount); @@ -178,7 +269,8 @@ internal sealed class MiniBroker : IDisposable private async Task ServeAsync(TcpClient client) { - var buffer = new byte[1024]; + var buffer = new byte[4096]; + var pending = new List(); try { var stream = client.GetStream(); @@ -195,25 +287,22 @@ internal sealed class MiniBroker : IDisposable continue; } - var packetType = buffer[0] & 0xF0; - if (packetType == 0x10) + // Frame properly: several packets can share one TCP read (concurrent publishes do), and + // one packet can span two. + pending.AddRange(buffer.AsSpan(0, read).ToArray()); + var close = false; + while (TryTakePacket(pending, out var packet)) { - await stream.WriteAsync(new byte[] { 0x20, 0x02, 0x00, ConnAckReturnCode }, _cts.Token); - - // A real broker closes the socket straight after refusing, which is what drives - // MQTTnet's DisconnectedAsync. Mirroring it keeps the fake honest. - if (ConnAckReturnCode != ReturnCodeAccepted) + if (!await HandlePacketAsync(stream, packet)) { + close = true; break; } } - else if (packetType == 0xC0) + + if (close) { - await stream.WriteAsync(PingResp, _cts.Token); - } - else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(buffer, read) is { } subAck) - { - await stream.WriteAsync(subAck, _cts.Token); + break; } } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 76d12cdd..6abfa293 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -288,6 +288,79 @@ public sealed class MqttDriverSessionLifecycleTests driver.GetHealth().State.ShouldBe(DriverState.Healthy); } + private static RawTagEntry WritableTag(string rawPath, string writeTopic) => new( + rawPath, + $$"""{"topic":"{{writeTopic}}/state","payloadFormat":"Scalar","dataType":"Float64","writeTopic":"{{writeTopic}}"}""", + WriteIdempotent: false); + + /// + /// R06:M-8. A write batch used to publish strictly serially, one awaited QoS 1 round-trip per + /// request — against a stalled broker, N writes × the publish deadline. Different topics now publish + /// concurrently: all four PUBLISHes reach a broker that never PUBACKs well inside ONE 30 s deadline + /// (serial code would have sent exactly one). Cancelling the batch then fills every slot, so the + /// batch shape holds. + /// + [Fact] + public async Task AWriteBatch_PublishesDifferentTopicsConcurrently() + { + using var broker = new MiniBroker(); // never PUBACKs + var options = BrokerOptions(broker.Port) with + { + ConnectTimeoutSeconds = 30, + RawTags = [.. Enumerable.Range(0, 4).Select(i => WritableTag($"Plant/Mqtt/dev1/Sp{i}", $"cmd/sp{i}"))], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var write = driver.WriteAsync( + [.. Enumerable.Range(0, 4).Select(i => new WriteRequest($"Plant/Mqtt/dev1/Sp{i}", (double)i))], + cts.Token); + + await WaitForAsync(() => broker.Published.Count == 4, "all four PUBLISHes to be in flight at once"); + write.IsCompleted.ShouldBeFalse("no PUBACK has arrived, so nothing can have completed"); + + await cts.CancelAsync(); + var results = await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + results.Count.ShouldBe(4); + results.ShouldAllBe(r => r.StatusCode == 0x80050000u); // BadCommunicationError, every slot filled + } + + /// + /// Writes to the SAME topic in one batch stay in request order — last write wins at the device — + /// while the batch as a whole is concurrent. The broker PUBACKs here, so every slot is Good — and + /// those acks advance LastSuccessfulRead (writes feed health). + /// + [Fact] + public async Task AWriteBatch_KeepsSameTopicWritesInRequestOrder() + { + using var broker = new MiniBroker { AckPublish = true }; + var options = BrokerOptions(broker.Port) with + { + RawTags = [WritableTag("Plant/Mqtt/dev1/Sp", "cmd/sp"), WritableTag("Plant/Mqtt/dev1/Other", "cmd/other")], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + driver.GetHealth().LastSuccessfulRead.ShouldBeNull("nothing has answered yet"); + + var results = await driver.WriteAsync( + [ + new WriteRequest("Plant/Mqtt/dev1/Sp", 1.0), + new WriteRequest("Plant/Mqtt/dev1/Other", 9.0), + new WriteRequest("Plant/Mqtt/dev1/Sp", 2.0), + new WriteRequest("Plant/Mqtt/dev1/Sp", 3.0), + ], + TestContext.Current.CancellationToken); + + results.ShouldAllBe(r => r.StatusCode == 0u); + broker.Published.Where(p => p.Topic == "cmd/sp") + .Select(p => System.Text.Encoding.UTF8.GetString(p.Payload)) + .ShouldBe(["1", "2", "3"]); + + // A PUBACK is the broker answering, so writes feed the evidence clock the comms-loss watchdog reads. + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); From 16f401b9e70f469a81463246bcfb47986881d218 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:46:21 -0400 Subject: [PATCH 16/34] fix(sql): dispose the poll reader's concurrency gate once no group holds it (R06:S-3) SqlPollReader's SemaphoreSlim was never disposed and a new reader is built per rebind. The reader is now IDisposable with a reference count (owner + one per in-flight group): the gate is disposed only when the last group - including a timed-out one still wedged in the provider - releases it, so nothing ever releases into a disposed semaphore. ApplyBinding disposes the replaced reader; the final DisposeAsync disposes the live one (Shutdown does not, since a re-init without a rebind keeps polling through it). Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 17 ++++- .../SqlPollReader.cs | 68 +++++++++++++++++-- .../SqlPollReaderTests.cs | 54 +++++++++++++++ .../SqlReinitConfigAdoptionTests.cs | 37 ++++++++++ 4 files changed, 171 insertions(+), 5 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 59ae4956..945fc32b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -261,6 +261,10 @@ public sealed class SqlDriver _factory = _explicitFactory ?? dialect.Factory; _connectionString = connectionString; Endpoint = DescribeEndpoint(connectionString); + + // The reader being replaced owns a SemaphoreSlim (R06:S-3). Disposing it is safe even with a stale + // poll still inside it: the gate goes only when the last in-flight group releases it. + var previous = _reader; _reader = new SqlPollReader( _factory, connectionString, @@ -271,6 +275,7 @@ public sealed class SqlDriver nullIsBad: options.NullIsBad, resolve: rawPath => _resolver.TryResolve(rawPath, out var definition) ? definition : null, logger: _logger); + previous?.Dispose(); } /// True when the supplied DriverConfig JSON carries a real body. The bootstrapper always passes @@ -1150,7 +1155,17 @@ public sealed class SqlDriver /// await using does not leak the poll loops. /// /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() => await TeardownAsync().ConfigureAwait(false); + public async ValueTask DisposeAsync() + { + await TeardownAsync().ConfigureAwait(false); + + // Only on the FINAL dispose: a Shutdown → Initialize cycle on the same instance keeps polling + // through this reader when the init does not rebind (no rebinder / placeholder config). + _reader.Dispose(); + } + + /// The live poll reader. Test observation of the S-3 lifecycle only. + internal SqlPollReader ReaderForTests => _reader; /// /// Shared teardown: stops every polled subscription and drops the authored table. Idempotent — safe diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index 04789c2d..da6ecfb5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -36,7 +36,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// no subscription state and no connection between polls, so the driver shell can wrap it in /// and own those concerns. /// -public sealed class SqlPollReader +public sealed class SqlPollReader : IDisposable { /// Minimum spacing between "your source violates the query contract" warnings, per reader. private static readonly TimeSpan ContractWarningInterval = TimeSpan.FromMinutes(1); @@ -52,12 +52,20 @@ public sealed class SqlPollReader /// /// Caps concurrently executing group queries — and therefore concurrently held connections - /// (design §8.4). Never disposed: a only needs disposal once its - /// AvailableWaitHandle has been touched, and not disposing it also removes the hazard of a - /// timed-out-but-still-running group releasing into a disposed semaphore. + /// (design §8.4). Disposed by only once every group that ever acquired it has + /// released it (), so a timed-out-but-still-running group can never release into a + /// disposed semaphore (R06:S-3 — it used to be never disposed, one per reinitialize). /// private readonly SemaphoreSlim _gate; + /// + /// Owner reference (1) plus one per in-flight group. drops the owner's; whoever + /// takes it to zero disposes . Zero is terminal — no group can start after it. + /// + private int _refs = 1; + + private int _disposed; + private long _lastContractWarningTicks; /// Initializes a new instance of the class. @@ -273,6 +281,22 @@ public sealed class SqlPollReader /// inside the deadline, so a poll behind a wedged group still returns on time — as BadTimeout. /// private async Task RunGroupAsync(SqlQueryPlan plan, CancellationToken cancellationToken) + { + // A reader replaced by a reinitialize (and disposed) can still be holding a stale poll: it runs no + // new group. Cancellation is the honest answer — the poll that asked has been superseded. + if (!TryAddRef()) throw new OperationCanceledException("The Sql poll reader was disposed."); + + try + { + return await RunGroupCoreAsync(plan, cancellationToken).ConfigureAwait(false); + } + finally + { + ReleaseRef(); + } + } + + private async Task RunGroupCoreAsync(SqlQueryPlan plan, CancellationToken cancellationToken) { var clock = Stopwatch.StartNew(); @@ -288,6 +312,9 @@ public sealed class SqlPollReader if (budget <= TimeSpan.Zero) return GroupResult.TimedOut; deadline.CancelAfter(budget); + // The work holds its OWN reference: it can outlive this method (a timed-out group is abandoned + // still running) and its finally releases the slot into _gate, which must therefore still exist. + AddRefForWork(); work = Task.Run( async () => { @@ -299,6 +326,7 @@ public sealed class SqlPollReader { _gate.Release(); deadline.Dispose(); + ReleaseRef(); } }, CancellationToken.None); @@ -334,6 +362,38 @@ public sealed class SqlPollReader } } + /// + /// Releases the concurrency gate once no group still holds it. Idempotent; safe while groups are in + /// flight (the last one out disposes it). A disposed reader starts no new group. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) ReleaseRef(); + } + + /// True once has actually been disposed. Test observation only. + internal bool GateDisposedForTests => Volatile.Read(ref _refs) == 0; + + /// Takes a reference unless the reader is already fully released. + private bool TryAddRef() + { + while (true) + { + var current = Volatile.Read(ref _refs); + if (current == 0) return false; + if (Interlocked.CompareExchange(ref _refs, current + 1, current) == current) return true; + } + } + + /// A further reference for work started under an already-held one (so never from zero). + private void AddRefForWork() => Interlocked.Increment(ref _refs); + + /// Drops a reference; the one that reaches zero disposes the gate. + private void ReleaseRef() + { + if (Interlocked.Decrement(ref _refs) == 0) _gate.Dispose(); + } + /// How much of this group's operationTimeout budget is left; never negative. private TimeSpan Remaining(Stopwatch clock) { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index da658517..87a0d02a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -522,6 +522,60 @@ public sealed class SqlPollReaderTests dialect.Factory, " ", dialect, CommandTimeout, OperationTimeout, 4, false, _ => null)); } + // ---- R06:S-3: the concurrency gate is released ---- + + /// An idle reader releases its gate on Dispose, and a disposed reader starts no new group. + [Fact] + public async Task Dispose_releasesTheGate_andADisposedReaderRunsNoNewGroup() + { + using var fixture = new SqlitePollFixture(); + var reader = Reader(fixture, KvTag("Speed", SqlitePollFixture.PresentKey)); + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + + reader.Dispose(); + reader.Dispose(); // idempotent + + reader.GateDisposedForTests.ShouldBeTrue(); + await Should.ThrowAsync(() => reader.ReadAsync(["Speed"], CancellationToken.None)); + } + + /// + /// Disposing while a timed-out group is still wedged inside the provider must NOT dispose the gate + /// under it — the zombie releases its slot into the gate when it finally finishes, which is exactly + /// the hazard that kept the gate undisposed before. The gate goes once the zombie is out. + /// + [Fact] + public async Task Dispose_withAWedgedGroupInFlight_waitsForItBeforeReleasingTheGate() + { + using var fixture = new SqlitePollFixture(); + var factory = new TrackingFactory(); + var reader = new SqlPollReader( + factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: TimeSpan.FromSeconds(30), operationTimeout: TimeSpan.FromMilliseconds(500), + maxConcurrentGroups: 1, nullIsBad: false, + resolve: Resolver(KvTag("Speed", SqlitePollFixture.PresentKey))); + + var locker = fixture.OpenNewConnection(); + Execute(locker, "BEGIN EXCLUSIVE"); + try + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + factory.Live.ShouldBe(1); // the zombie is still inside the provider — the control + + reader.Dispose(); + reader.GateDisposedForTests.ShouldBeFalse("a group still holds the gate"); + } + finally + { + Execute(locker, "ROLLBACK"); + locker.Dispose(); + } + + (await WaitUntilAsync(() => reader.GateDisposedForTests, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("the last group out must release the gate"); + factory.Live.ShouldBe(0); + } + [Fact] public async Task ReadAsync_rejectsANullReferenceList() => await Should.ThrowAsync(async () => diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs index 1181de1a..f9223f94 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs @@ -101,6 +101,43 @@ public sealed class SqlReinitConfigAdoptionTests driver.Endpoint.ShouldContain("old-server"); } + /// + /// R06:S-3. Every rebind builds a new ; the replaced one (and its + /// SemaphoreSlim) used to be dropped undisposed, one per reinitialize. The control: the reader + /// that replaced it is live. + /// + [Fact] + public async Task A_rebind_disposes_the_reader_it_replaces() + { + using var _ = new ScopedEnvironmentVariable( + SqlConnectionStringResolver.EnvironmentVariablePrefix + RefA, + "Server=old-server;Database=OldDb;Integrated Security=true"); + using var __ = new ScopedEnvironmentVariable( + SqlConnectionStringResolver.EnvironmentVariablePrefix + RefB, + "Server=new-server;Database=NewDb;Integrated Security=true"); + + var driver = SqlDriverFactoryExtensions.CreateInstance( + "sql-1", $$"""{"provider":"SqlServer","connectionStringRef":"{{RefA}}"}""", loggerFactory: null); + var original = driver.ReaderForTests; + + try + { + await driver.ReinitializeAsync( + $$"""{"provider":"SqlServer","connectionStringRef":"{{RefB}}"}""", CancellationToken.None); + } + catch + { + // Liveness failure against a non-existent server is expected; the rebind ran first. + } + + driver.ReaderForTests.ShouldNotBeSameAs(original); + original.GateDisposedForTests.ShouldBeTrue(); + driver.ReaderForTests.GateDisposedForTests.ShouldBeFalse(); + + await driver.DisposeAsync(); + driver.ReaderForTests.GateDisposedForTests.ShouldBeTrue("the final dispose releases the live reader too"); + } + /// Sets an environment variable for the duration of a test and restores it after. The resolver /// reads process-global state, so a leaked variable would race every sibling test. private sealed class ScopedEnvironmentVariable : IDisposable From 91b7e9541b1d230907475396006328123af7790d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:47:18 -0400 Subject: [PATCH 17/34] fix(sql): withhold the JSON parser's text on a malformed driver config (R06:S-8) The factory quoted System.Text.Json's message (and attached the JsonException) into the failure that reaches InitializeFailed -> health LastError -> /hosts; a pasted connection-string literal is exactly the malformed text that lands there. The message now names the instance and the line/byte position only, as the probe and browser already do. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlDriverFactoryExtensions.cs | 8 ++++++- .../SqlDriverFactoryTests.cs | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs index a53e4390..3bb727e5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs @@ -194,8 +194,14 @@ public static class SqlDriverFactoryExtensions } catch (JsonException ex) { + // The parser's message AND the exception carrying it are withheld (R06:S-8), matching + // SqlDriverProbe and SqlDriverBrowser: System.Text.Json quotes the offending token, and a pasted + // connection-string literal is exactly the malformed text that lands here. This message becomes + // InitializeFailed -> health LastError -> /hosts. Line/byte position carry no content, so they stay. throw new InvalidOperationException( - $"Sql driver config for '{driverInstanceId}' is not valid JSON: {ex.Message}", ex); + $"Sql driver config for '{driverInstanceId}' is not valid JSON (line {ex.LineNumber + 1}, byte " + + $"{ex.BytePositionInLine}). The parser's message is withheld because the malformed text may " + + "carry a connection string."); } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs index 65d6a281..9d1265de 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs @@ -25,6 +25,28 @@ public sealed class SqlDriverFactoryTests { // ---- the plan's headline legs ---- + /// + /// R06:S-8. A malformed config used to quote System.Text.Json's message — which echoes the + /// offending token — into the exception that becomes InitializeFailed → health + /// LastError → /hosts. A pasted connection string is exactly the malformed text that + /// reaches here, so the parser text (and the inner exception carrying it) is withheld, as the probe and + /// browser already do. The message still names the instance and says what is wrong. + /// + [Fact] + public void CreateInstance_malformedJson_withholdsTheParserText() + { + const string secret = "hunter2-do-not-log"; + var ex = Should.Throw(() => + SqlDriverFactoryExtensions.CreateInstance( + "d1", $$"""{"provider":"SqlServer","connectionStringRef": Password={{secret}} }""", null)); + + ex.Message.ShouldContain("d1"); + ex.Message.ShouldContain("not valid JSON"); + ex.Message.ShouldNotContain(secret); + ex.Message.ShouldNotContain("Password"); + ex.InnerException.ShouldBeNull(); + } + [Fact] public void CreateInstance_missingConnectionStringRef_throwsActionable() => Should.Throw(() => From 7f5d959fb938662509bd14dda1454ae0bf739382 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:49:24 -0400 Subject: [PATCH 18/34] feat(sql): surface IS_NULLABLE on browsed column attributes (R06:S-5) Every dialect projects IS_NULLABLE and nothing read it. The browse session now reads it (tolerantly: a dialect that does not project it reads as unknown) and the picker's descriptor line says 'nullable' / 'NOT NULL', so an operator knows a cell can come back NULL (Uncertain, or Bad under nullIsBad) before committing the tag. The runtime catalog loader keeps reading names only - nothing at runtime consumes nullability. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlBrowseSession.cs | 43 +++++++++++++++++-- .../SqlBrowseSessionTests.cs | 8 +++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs index f25aaff7..7feb6a0f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs @@ -48,6 +48,12 @@ internal sealed class SqlBrowseSession : IBrowseSession /// Column alias carrying the SQL type family name. private const string DataTypeColumn = "DATA_TYPE"; + /// + /// Column alias carrying YES/NO nullability. Every shipped dialect projects it (Oracle + /// normalises its Y/N); it used to be read by nothing (R06:S-5). + /// + private const string IsNullableColumn = "IS_NULLABLE"; + /// The TABLE_TYPE value marking a view rather than a base table. private const string ViewTableType = "VIEW"; @@ -158,7 +164,7 @@ internal sealed class SqlBrowseSession : IBrowseSession reference.Schema, reference.Table!, cancellationToken).ConfigureAwait(false); // Computed from the columns ALREADY READ for this call — the catalog is not queried a second time. - var timestampColumn = PreferredTimestampColumn(columns); + var timestampColumn = PreferredTimestampColumn([.. columns.Select(c => (c.Name, c.DataType))]); // Ordinal: a column NodeId is only ever minted from this same catalog output, so its case is the // catalog's own. A case-insensitive match would pick the wrong column on a case-sensitive collation @@ -191,7 +197,15 @@ internal sealed class SqlBrowseSession : IBrowseSession Name: column.Name, DriverDataType: _dialect.MapColumnType(column.DataType).ToString(), IsArray: false, - SecurityClass: ReadOnlySecurityClass, + // The picker's descriptor line ("{type} · {SecurityClass}") — display only. Nullability is + // what tells an operator a cell can come back NULL, which publishes Uncertain (or Bad under + // nullIsBad), before they commit the tag. + SecurityClass: column.IsNullable switch + { + true => ReadOnlySecurityClass + " · nullable", + false => ReadOnlySecurityClass + " · NOT NULL", + null => ReadOnlySecurityClass, + }, IsAlarm: false, AddressFields: addressFields), ]; @@ -356,7 +370,7 @@ internal sealed class SqlBrowseSession : IBrowseSession /// than an error: the catalog answering "no rows" is indistinguishable from a genuinely column-less /// relation, and neither is a reason to fail an operator's click. /// - private Task> ReadColumnsAsync( + private Task> ReadColumnsAsync( string schema, string table, CancellationToken ct) => QueryAsync( _dialect.ListColumnsSql, @@ -367,9 +381,30 @@ internal sealed class SqlBrowseSession : IBrowseSession }, static reader => ( Name: ReadString(reader, ColumnNameColumn), - DataType: ReadOptionalString(reader, DataTypeColumn) ?? string.Empty), + DataType: ReadOptionalString(reader, DataTypeColumn) ?? string.Empty, + IsNullable: ReadNullability(reader)), ct); + /// + /// Reads IS_NULLABLE as a tri-state: when a (custom) dialect does not + /// project it or the value is neither YES nor NO — "unknown" is said by omission, never + /// guessed. + /// + private static bool? ReadNullability(DbDataReader reader) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (!string.Equals(reader.GetName(i), IsNullableColumn, StringComparison.OrdinalIgnoreCase)) continue; + if (reader.IsDBNull(i)) return null; + var text = reader.GetValue(i)?.ToString()?.Trim(); + if (string.Equals(text, "YES", StringComparison.OrdinalIgnoreCase)) return true; + if (string.Equals(text, "NO", StringComparison.OrdinalIgnoreCase)) return false; + return null; + } + + return null; + } + /// /// Runs one catalog query under the gate and projects every row. The disposed check happens /// inside the gate wait so a dispose racing an in-flight call cannot be missed. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs index 6be390f9..0bd53ad5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs @@ -78,7 +78,8 @@ public sealed class SqlBrowseSessionTests attribute.Name.ShouldBe(SqliteBrowseFixture.RealColumn); attribute.DriverDataType.ShouldBe(nameof(DriverDataType.Float64)); attribute.IsArray.ShouldBeFalse(); - attribute.SecurityClass.ShouldBe("ViewOnly"); + // R06:S-5: every dialect projects IS_NULLABLE and the picker now says so. The column is declared NULL. + attribute.SecurityClass.ShouldBe("ViewOnly · nullable"); attribute.IsAlarm.ShouldBeFalse(); } @@ -92,7 +93,10 @@ public sealed class SqlBrowseSessionTests SqliteDialect.MainSchema, SqliteBrowseFixture.KeyValueTable, SqliteBrowseFixture.TextColumn); var attributes = await session.AttributesAsync(nodeId, CancellationToken.None); - attributes.ShouldHaveSingleItem().DriverDataType.ShouldBe(nameof(DriverDataType.String)); + var attribute = attributes.ShouldHaveSingleItem(); + attribute.DriverDataType.ShouldBe(nameof(DriverDataType.String)); + // The NOT NULL control for the nullable case above — the descriptor tracks the catalog, not a constant. + attribute.SecurityClass.ShouldBe("ViewOnly · NOT NULL"); } /// From fd6ecf8293536e24d5c1ef8a05e299bde207abd8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:55:32 -0400 Subject: [PATCH 19/34] fix(modbus): review r0 - FC23 opt-in, explicit maxReadGap 0 = off, legacy coil types load, one health writer Coordinator rulings and Important findings from review r0: 1. FC23 is opt-in (new UseFC23ForMultiRegisterWrites, default false); a config without it - including the persisted "disableFC23": false every form-saved config carries - keeps the FC16 wire. DisableFC23 stays the hard kill-switch. 2. An explicit maxReadGap 0 means coalescing off again (its pre-lane meaning); the gap-8 default applies only to an absent key. 3. Coils/DiscreteInputs tags persisted with a register dataType (the old editor saved Int16) are coerced to Bool, never refused, with no deploy warning (a warning can be promoted to a blocking error). The editor loads them as Bool. Register region + Bool is still refused. 4. One comms-fault slot for reads and writes: a good read clears a write's transport fault and vice versa. Minors: Dispose writes its own Unknown health (m1); persisted region/address are authoritative over addressString so a Family change cannot remap a live tag (m2); a malformed-but-delivered reply no longer fails the unit fast (m3); a fresh socket published after a concurrent dispose is torn down (m4); the editor resets address text per tag (m5); master merged (m6); the FC23 latch log says re-init restores it (m7). Back-compat tests use the old persisted JSON shapes and fail without the fixes. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ModbusDriverOptions.cs | 37 +++++---- .../ModbusTagDefinitionFactory.cs | 45 +++++----- .../ModbusDriver.cs | 82 +++++++++++-------- .../ModbusDriverFactoryExtensions.cs | 3 + .../ModbusSocketLifecycle.cs | 7 ++ .../Drivers/Forms/ModbusDriverForm.razor | 13 ++- .../TagEditors/ModbusTagConfigEditor.razor | 5 +- .../Uns/TagEditors/ModbusTagConfigModel.cs | 16 ++-- .../CommsRecoveryTests.cs | 14 ++-- .../ModbusCoalescingDefaultsTests.cs | 26 ++++++ .../ModbusCoalescingTests.cs | 6 +- .../ModbusFamilyWiringTests.cs | 73 +++++++++++++---- .../ModbusHealthContractTests.cs | 43 +++++++++- .../ModbusRegionTypeValidationTests.cs | 41 +++++++++- .../Uns/ModbusTagConfigModelTests.cs | 20 ++++- 15 files changed, 314 insertions(+), 117 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs index 4f456585..fa24f271 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusDriverOptions.cs @@ -89,16 +89,24 @@ public sealed class ModbusDriverOptions public bool UseFC16ForSingleRegisterWrites { get; init; } = false; /// - /// Kill-switch for FC23 (Read/Write Multiple Registers). By default a multi-register holding - /// write (2–121 registers: 32/64-bit values, arrays, strings) is sent as ONE FC23 that writes - /// the block and reads the same block back, so the device confirms what it now holds. A unit - /// that answers FC23 with exception 01 (MELSEC QJ71MT91, S7 Modbus servers) is moved to FC16 - /// automatically, for that write and every later one; one that fails FC23 at the transport is - /// moved too (that one write is reported as a comms failure). Set true to always use - /// FC16. Default false. + /// Hard kill-switch for FC23 (Read/Write Multiple Registers): when true the driver never + /// emits FC23, even with set. Default false. + /// FC23 is OPT-IN — a config without keeps the FC16 wire + /// behaviour, so the persisted "disableFC23": false every form-saved config carries changes + /// nothing on the wire. /// public bool DisableFC23 { get; init; } = false; + /// + /// Opt-in: send a multi-register holding write (2–121 registers: 32/64-bit values, arrays, strings) + /// as ONE FC23 that writes the block and reads the same block back, so the device confirms what it + /// now holds. A unit that answers FC23 with exception 01 (MELSEC QJ71MT91, S7 Modbus servers) moves + /// to FC16 for that write and every later one until the next initialize; one that fails FC23 at the + /// transport moves too (that one write is reported as a comms failure). Ignored when + /// is set. Default false = FC16, the historical behaviour. + /// + public bool UseFC23ForMultiRegisterWrites { get; init; } = false; + /// /// Interval for the background re-probe loop that retries auto-prohibited /// coalesced ranges. When non-null, every AutoProhibitReprobeInterval @@ -110,14 +118,13 @@ public sealed class ModbusDriverOptions public TimeSpan? AutoProhibitReprobeInterval { get; init; } = null; /// - /// Block-read coalescing budget. The read planner combines tags in the same (UnitId, Region) - /// group whose addresses are at most this many registers (Coils / DiscreteInputs: bits) apart - /// into a single FC03/FC04/FC01/FC02 read, then slices the response back into per-tag values. - /// 0 coalesces only adjacent or overlapping tags (several BitInRegister tags on one - /// status word still fold into one read). Default 8. A range the device refuses is - /// auto-prohibited and read per tag, so a protected hole costs one extra round trip, not data. To - /// turn coalescing off entirely set , or exempt one tag with - /// its coalesceProhibited key. + /// Block-read coalescing budget. When non-zero, the read planner combines tags in the same + /// (UnitId, Region) group whose addresses are at most this many registers (Coils / DiscreteInputs: + /// bits) apart into a single FC03/FC04/FC01/FC02 read, then slices the response back into per-tag + /// values (several BitInRegister tags on one status word fold into one read). 0 = no + /// coalescing — an explicitly authored 0 keeps that historical meaning. Default 8 applies + /// only when the key is ABSENT. A range the device refuses is auto-prohibited and read per tag. + /// also turns it off; coalesceProhibited exempts one tag. /// public ushort MaxReadGap { get; init; } = DefaultMaxReadGap; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs index d34c55ba..633d6931 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs @@ -35,12 +35,14 @@ public static class ModbusTagDefinitionFactory => FromTagConfig(tagConfig, rawPath, ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out def); /// - /// Family-aware overload: an addressString key (the Modicon / mnemonic / family-native grammar - /// of — 40001:F:CDAB, DL205 V2000, MELSEC D100) - /// is resolved with the DRIVER's / and is - /// authoritative for region, offset, type, bit, string length, byte order and array count; the - /// editor also persists the resolved region / address keys beside it for display and for - /// readers that predate it. Without an addressString this is exactly the 3-argument overload. + /// Family-aware overload. The persisted region + address keys are AUTHORITATIVE whenever + /// present — the editor resolves an addressString with the driver's family and persists the + /// result, so a later Family / MelsecSubFamily change can never silently remap a live tag. A blob that + /// carries ONLY an addressString (CSV / API authoring, the Modicon / mnemonic / family-native + /// grammar of — 40001:F:CDAB, DL205 V2000, MELSEC + /// D100) is resolved here with the DRIVER's / + /// . Without an addressString this is exactly the 3-argument + /// overload. /// /// The authored equipment-tag TagConfig JSON. /// The tag's RawPath — becomes the definition's identity (Name). @@ -67,8 +69,9 @@ public static class ModbusTagDefinitionFactory ushort stringLength; int? arrayCount; var addressText = ReadAddressString(root, out var addressStringInvalid); - if (addressStringInvalid) return false; - if (addressText is not null) + var hasResolvedKeys = root.TryGetProperty("region", out _) && root.TryGetProperty("address", out _); + if (!hasResolvedKeys && addressStringInvalid) return false; + if (!hasResolvedKeys && addressText is not null) { if (!ModbusAddressParser.TryParse(addressText, family, melsecSubFamily, out var parsed, out _) || parsed is null) return false; @@ -106,9 +109,12 @@ public static class ModbusTagDefinitionFactory var arrayLength = ReadInt(root, "arrayLength"); arrayCount = (isArray && arrayLength >= 1) ? arrayLength : null; } - // Region/type compatibility: a bit region carries only Bool, a register region never Bool. - // A mismatch used to throw out of the coalescing planner (killing every tag's poll) or - // silently return a bool into an Int32 node. + // Region/type compatibility. A bit region (Coils / DiscreteInputs) carries only Bool: a non-Bool + // dataType there is COERCED to Bool, never refused — the pre-#741 editor defaulted dataType to + // Int16 and persisted it for coil tags, and the old driver ignored dataType for bit regions, so + // those tags read and wrote correctly and must keep doing so. A register region with Bool is + // refused: that pair never read (it threw out of the planner and killed every tag's poll). + if (region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs) dataType = ModbusDataType.Bool; if (!IsRegionTypeCompatible(region, dataType)) return false; if (!TagConfigJson.TryReadEnumStrict(root, "stringByteOrder", ModbusStringByteOrder.HighByteFirst, out var stringByteOrder)) return false; // Guard: String tags require StringLength >= 1. RegisterCount = (StringLength+1)/2, @@ -314,14 +320,15 @@ public static class ModbusTagDefinitionFactory private static void DescribeRejection(JsonElement root, List warnings) { var addressText = ReadAddressString(root, out var addressStringInvalid); - if (addressStringInvalid) + var hasResolvedKeys = root.TryGetProperty("region", out _) && root.TryGetProperty("address", out _); + if (!hasResolvedKeys && addressStringInvalid) { warnings.Add("Modbus TagConfig 'addressString' is not a string — the tag will not resolve (BadNodeIdUnknown)."); return; } - if (addressText is not null) + if (!hasResolvedKeys && addressText is not null) { - // The addressString is authoritative; the other address keys are not read at all. + // String-only blob: the addressString is what the driver resolves. if (!ModbusAddressParser.TryParse(addressText, out _, out var error)) warnings.Add($"Modbus 'addressString' '{addressText}' does not parse under any PLC family ({error}) — " + "the tag will not resolve (BadNodeIdUnknown)."); @@ -372,12 +379,12 @@ public static class ModbusTagDefinitionFactory ? r : ModbusRegion.HoldingRegisters; if (!TagConfigJson.TryReadEnumStrict(root, "dataType", DefaultDataType(region), out var dataType)) return; - if (root.TryGetProperty("region", out _) && !IsRegionTypeCompatible(region, dataType)) + // Only a register region with Bool is refused (a bit region with any type is coerced to Bool). + if (root.TryGetProperty("region", out _) && region is not (ModbusRegion.Coils or ModbusRegion.DiscreteInputs) + && !IsRegionTypeCompatible(region, dataType)) { - warnings.Add(region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs - ? $"Modbus region {region} only carries Bool tags; got {dataType} — the tag is rejected (BadNodeIdUnknown)." - : $"Modbus Bool tags only live in Coils / DiscreteInputs; a {region} tag must use a register type " - + "(use BitInRegister for a bit of a register) — the tag is rejected (BadNodeIdUnknown)."); + warnings.Add($"Modbus Bool tags only live in Coils / DiscreteInputs; a {region} tag must use a register type " + + "(use BitInRegister for a bit of a register) — the tag is rejected (BadNodeIdUnknown)."); } if (dataType == ModbusDataType.String && (ushort)ReadInt(root, "stringLength") < 1) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs index ad9b7295..83a28405 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriver.cs @@ -61,11 +61,12 @@ public sealed class ModbusDriver /// Guards , the transport/table publish and the health fields. private readonly object _stateLock = new(); - // Health is composed from two independent sides so a good write clears only a WRITE-side - // Degraded (and a good read only a read-side one). LastSuccessfulRead advances on any evidence the - // device answered — a good read, a good write, a refusal, or a probe answer. - private string? _readFault; - private string? _writeFault; + // ONE comms-fault slot for reads AND writes: a fault is a property of the shared transport, so the + // latest batch of either kind decides — a good read clears a Degraded a write's timeout raised and vice + // versa (two independent sides latched a single write blip as Degraded forever in a read-mostly plant). + // LastSuccessfulRead advances on any evidence the device answered — a good read, a good write, a + // refusal, or a probe answer. + private string? _commsFault; // Once-per-tag Warning for deterministic device refusals (exception 01/02/03, undecodable data). // Replaced (never cleared in place) on every table publish. @@ -246,16 +247,13 @@ public sealed class ModbusDriver // failure is the re-initialize window itself, which the lifecycle reports on its own. var current = _health; if (current.State is DriverState.Faulted or DriverState.Initializing or DriverState.Unknown) return; - _readFault = ex.Message; + _commsFault = ex.Message; PublishComposedHealthLocked(current.LastSuccessfulRead); } } // ---- health composition ---- - /// Which half of the driver a batch outcome came from. - private enum HealthSide { Read, Write } - /// /// Per-batch accounting: did the device answer anything, did a comms fault happen, and which units /// (or the whole endpoint) must the rest of this batch fail fast against. @@ -277,7 +275,7 @@ public sealed class ModbusDriver /// is still current. A batch that touched no device (every reference /// unknown, every write refused before the wire) changes nothing. /// - private void ReportOutcome(int generation, HealthSide side, BatchOutcome batch, DateTime now) + private void ReportOutcome(int generation, BatchOutcome batch, DateTime now) { if (!batch.Answered && batch.CommsFault is null) return; lock (_stateLock) @@ -285,16 +283,15 @@ public sealed class ModbusDriver if (generation != _generation) return; var current = _health; if (current.State is DriverState.Faulted or DriverState.Initializing or DriverState.Unknown) return; - if (side == HealthSide.Read) _readFault = batch.CommsFault; - else _writeFault = batch.CommsFault; + _commsFault = batch.CommsFault; PublishComposedHealthLocked(batch.Answered ? now : current.LastSuccessfulRead); } } - /// Writes Healthy/Degraded from the two fault sides. Caller holds . + /// Writes Healthy/Degraded from the comms-fault slot. Caller holds . private void PublishComposedHealthLocked(DateTime? lastSuccessfulRead) { - var fault = _readFault ?? _writeFault; + var fault = _commsFault; WriteHealth(fault is null ? new DriverHealth(DriverState.Healthy, lastSuccessfulRead, null) : new DriverHealth(DriverState.Degraded, lastSuccessfulRead, fault)); @@ -321,8 +318,7 @@ public sealed class ModbusDriver lock (_stateLock) { if (generation != _generation) return; - _readFault = null; - _writeFault = null; + _commsFault = null; WriteHealth(health); } } @@ -370,7 +366,10 @@ public sealed class ModbusDriver return StatusBadCommunicationError; default: batch.CommsFault ??= ex.Message; - batch.FailUnit(unit); + // Fail the unit fast only when nothing came back in time (timeout / desync / socket-level IO): + // a malformed-but-delivered reply (byte-count mismatch) proves the unit responds, so its + // siblings are still tried. + if (ex is IOException or TimeoutException or OperationCanceledException) batch.FailUnit(unit); return StatusBadCommunicationError; } } @@ -493,8 +492,7 @@ public sealed class ModbusDriver generation = ++_generation; previous = _transport; _transport = null; - _readFault = null; - _writeFault = null; + _commsFault = null; WriteHealth(new DriverHealth(DriverState.Initializing, null, null)); } @@ -728,7 +726,9 @@ public sealed class ModbusDriver var results = new DataValueSnapshot[fullReferences.Count]; var batch = new BatchOutcome(); - var coalesced = !_options.DisableReadCoalescing + // MaxReadGap 0 = coalescing OFF (its pre-lane meaning, kept for every persisted config that authored + // it); the gap-8 default applies only to an absent key. + var coalesced = !_options.DisableReadCoalescing && _options.MaxReadGap > 0 ? await ReadCoalescedAsync(transport, table, fullReferences, results, now, batch, cancellationToken).ConfigureAwait(false) : new HashSet(); @@ -774,7 +774,7 @@ public sealed class ModbusDriver // ONE health write per batch, worst-wins over the batch, generation-checked. It used to be // last-writer-wins per tag, so health flapped with tag order. - ReportOutcome(generation, HealthSide.Read, batch, now); + ReportOutcome(generation, batch, now); return results; } @@ -1503,9 +1503,9 @@ public sealed class ModbusDriver } } - // Write-side health: a comms failure degrades it (LastSuccessfulRead preserved); a good write or a - // device refusal proves the device answered and clears a write-side Degraded. Generation-checked. - ReportOutcome(generation, HealthSide.Write, batch, DateTime.UtcNow); + // Write outcome feeds the same single health slot as reads: a comms failure degrades (LastSuccessfulRead + // preserved); a good write or a device refusal proves the device answered and clears it. Generation-checked. + ReportOutcome(generation, batch, DateTime.UtcNow); return results; } @@ -1665,7 +1665,8 @@ public sealed class ModbusDriver /// /// Multi-register write via FC23 (Read/Write Multiple Registers): the device writes the block and /// returns the SAME block read back in one transaction, so the write is confirmed by the device's own - /// post-write registers. Gated by . A device that does + /// post-write registers. OPT-IN via , + /// and is the hard kill-switch. A device that does /// not implement FC23 (MELSEC QJ71MT91, S7 Modbus servers — exception 01) falls back to FC16 for this /// write and every later one on that unit, which is safe because an Illegal Function wrote nothing. /// A transport failure on FC23 is reported (its outcome is unknown, so it is not retried as FC16) and @@ -1681,7 +1682,8 @@ public sealed class ModbusDriver var unsupported = Volatile.Read(ref _fc23Unsupported); // Only genuinely multi-register writes: a single register forced onto FC16 (UseFC16ForSingleRegisterWrites) // stays on the function code the operator named. - if (_options.DisableFC23 || qty < 2 || qty > Fc23MaxWriteRegisters || unsupported.ContainsKey(unit)) return false; + if (!_options.UseFC23ForMultiRegisterWrites || _options.DisableFC23 + || qty < 2 || qty > Fc23MaxWriteRegisters || unsupported.ContainsKey(unit)) return false; var pdu = new byte[10 + 1 + bytes.Length]; pdu[0] = 0x17; @@ -1701,7 +1703,7 @@ public sealed class ModbusDriver { if (unsupported.TryAdd(unit, 0)) _logger.LogInformation( - "Modbus unit {Unit} does not implement FC23 (Illegal Function); multi-register writes use FC16. Driver={DriverInstanceId}", + "Modbus unit {Unit} does not implement FC23 (Illegal Function); multi-register writes use FC16 until the driver is re-initialized. Driver={DriverInstanceId}", unit, _driverInstanceId); return false; } @@ -1709,7 +1711,7 @@ public sealed class ModbusDriver { if (unsupported.TryAdd(unit, 0)) _logger.LogWarning(ex, - "Modbus FC23 write to unit {Unit} failed at the transport; later multi-register writes to it use FC16. Driver={DriverInstanceId}", + "Modbus FC23 write to unit {Unit} failed at the transport; later multi-register writes to it use FC16 until the driver is re-initialized (a redeploy or reconnect restores FC23). Driver={DriverInstanceId}", unit, _driverInstanceId); throw; } @@ -2318,7 +2320,13 @@ public sealed class ModbusDriver // Dispose supersedes any in-flight (abandoned) initialize up front, even when it then has to // bypass the gate: that initialize will find its generation moved on and release its own // transport instead of publishing it after this teardown. - lock (_stateLock) _generation++; + int generation; + DateTime? lastRead; + lock (_stateLock) + { + generation = ++_generation; + lastRead = _health.LastSuccessfulRead; + } if (await _lifecycleGate.WaitAsync(LifecycleGateTimeout).ConfigureAwait(false)) { try @@ -2329,14 +2337,18 @@ public sealed class ModbusDriver { _lifecycleGate.Release(); } - - return; + } + else + { + _logger.LogWarning( + "Modbus lifecycle gate still held at dispose; tearing the transport down anyway. Driver={DriverInstanceId}", + _driverInstanceId); + await TeardownAsync().ConfigureAwait(false); } - _logger.LogWarning( - "Modbus lifecycle gate still held at dispose; tearing the transport down anyway. Driver={DriverInstanceId}", - _driverInstanceId); - await TeardownAsync().ConfigureAwait(false); + // Dispose's own final health write. Bumping the generation above drops a still-running shutdown's + // Unknown write, so without this a disposed driver kept reporting whatever it last was (e.g. Healthy). + WriteLifecycleHealth(generation, new DriverHealth(DriverState.Unknown, lastRead, null)); } /// diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs index 0886124a..e57ef5e0 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusDriverFactoryExtensions.cs @@ -85,6 +85,7 @@ public static class ModbusDriverFactoryExtensions UseFC15ForSingleCoilWrites = dto.UseFC15ForSingleCoilWrites ?? false, UseFC16ForSingleRegisterWrites = dto.UseFC16ForSingleRegisterWrites ?? false, DisableFC23 = dto.DisableFC23 ?? false, + UseFC23ForMultiRegisterWrites = dto.UseFC23ForMultiRegisterWrites ?? false, WriteOnChangeOnly = dto.WriteOnChangeOnly ?? false, MaxReadGap = dto.MaxReadGap ?? ModbusDriverOptions.DefaultMaxReadGap, DisableReadCoalescing = dto.DisableReadCoalescing ?? false, @@ -187,6 +188,8 @@ public static class ModbusDriverFactoryExtensions public bool? UseFC16ForSingleRegisterWrites { get; init; } /// Gets or sets a value indicating whether to disable function code 23. public bool? DisableFC23 { get; init; } + /// Gets or sets a value indicating whether multi-register writes opt in to FC23 write + read-back. + public bool? UseFC23ForMultiRegisterWrites { get; init; } /// Gets or sets a value indicating whether to write only on change. public bool? WriteOnChangeOnly { get; init; } /// Gets or sets the maximum gap between register addresses for coalescing. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs index 34fcbdd3..3cb4fe82 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus/ModbusSocketLifecycle.cs @@ -103,6 +103,13 @@ public sealed class ModbusSocketLifecycle : IAsyncDisposable _stream = client.GetStream(); _client = client; client = null; // ownership transferred + if (Volatile.Read(ref _disposed) != 0) + { + // A DisposeAsync that ran between the check above and the publish has already torn down; + // tear the fresh socket down ourselves so it is never stranded for the finalizer. + await TearDownAsync().ConfigureAwait(false); + throw new ObjectDisposedException(nameof(ModbusSocketLifecycle)); + } _lastSuccessUtc = DateTime.UtcNow; } catch (OperationCanceledException) when (!ct.IsCancellationRequested) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor index ce1d77fc..d3325e34 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Drivers/Forms/ModbusDriverForm.razor @@ -80,7 +80,7 @@
-
Default 8. 0 = adjacent tags only. Refused ranges fall back per tag.
+
0 = no coalescing. Default 8. Refused ranges fall back per tag.
@@ -112,7 +112,13 @@
- + +
+
+
+
+ +
@@ -324,6 +330,7 @@ public bool UseFC15ForSingleCoilWrites { get; set; } = false; public bool UseFC16ForSingleRegisterWrites { get; set; } = false; public bool DisableFC23 { get; set; } = false; + public bool UseFC23ForMultiRegisterWrites { get; set; } = false; public bool WriteOnChangeOnly { get; set; } = false; // Keep-alive @@ -368,6 +375,7 @@ UseFC15ForSingleCoilWrites = o.UseFC15ForSingleCoilWrites, UseFC16ForSingleRegisterWrites = o.UseFC16ForSingleRegisterWrites, DisableFC23 = o.DisableFC23, + UseFC23ForMultiRegisterWrites = o.UseFC23ForMultiRegisterWrites, WriteOnChangeOnly = o.WriteOnChangeOnly, KeepAliveEnabled = o.KeepAlive.Enabled, KeepAliveTimeSeconds = (int)o.KeepAlive.Time.TotalSeconds, @@ -404,6 +412,7 @@ UseFC15ForSingleCoilWrites = UseFC15ForSingleCoilWrites, UseFC16ForSingleRegisterWrites = UseFC16ForSingleRegisterWrites, DisableFC23 = DisableFC23, + UseFC23ForMultiRegisterWrites = UseFC23ForMultiRegisterWrites, AutoProhibitReprobeInterval = AutoProhibitReprobeIntervalSeconds > 0 ? TimeSpan.FromSeconds(AutoProhibitReprobeIntervalSeconds) : null, MaxReadGap = (ushort)Math.Clamp(MaxReadGap, 0, 65535), diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor index 86842227..33eae3fe 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor @@ -80,7 +80,10 @@ if (ConfigJson == _lastConfigJson) { return; } _lastConfigJson = ConfigJson; _m = ModbusTagConfigModel.FromJson(ConfigJson); - _addressText ??= _m.AddressString; + // Reset on EVERY config change: a reused editor instance switched to another tag must not show the + // previous tag's address text. + _addressText = _m.AddressString; + _addressError = null; _family = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()).Family; } diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs index 3ce00ad8..25f1efdd 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs @@ -32,9 +32,10 @@ public sealed class ModbusTagConfigModel /// /// The address in the parser grammar (40001:F:CDAB, DL205 V2000, MELSEC D100) when - /// the tag was authored that way; null for a tag authored by region + numeric address. When - /// present the driver re-resolves it with ITS family at runtime, so it is authoritative — editing any - /// resolved field by hand clears it (). + /// the tag was authored that way; null for a tag authored by region + numeric address. The + /// persisted resolved fields are what the driver reads (the string is only resolved by the driver for a + /// string-only blob), so editing any resolved field by hand clears it () + /// to keep the two from disagreeing. /// public string? AddressString { get; set; } @@ -130,9 +131,12 @@ public sealed class ModbusTagConfigModel _source = json ?? "", Region = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters), Address = TagConfigJson.GetInt(o, "address"), - // Absent dataType defaults per region, exactly as the driver's factory does (Bool for a bit region). - DataType = TagConfigJson.GetEnum(o, "dataType", - ModbusTagDefinitionFactory.DefaultDataType(TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters))), + // Absent dataType defaults per region, exactly as the driver's factory does (Bool for a bit region), + // and a bit region shows Bool whatever was persisted — the driver coerces a legacy coil/DI + // "dataType":"Int16" (what the pre-#741 editor saved) to Bool, so the editor shows what is polled. + DataType = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters) is ModbusRegion.Coils or ModbusRegion.DiscreteInputs + ? ModbusDataType.Bool + : TagConfigJson.GetEnum(o, "dataType", ModbusDataType.Int16), ByteOrder = TagConfigJson.GetEnum(o, "byteOrder", ModbusByteOrder.BigEndian), BitIndex = TagConfigJson.GetInt(o, "bitIndex"), StringLength = TagConfigJson.GetInt(o, "stringLength"), diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs index 7aeefb70..eaf68b33 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.IntegrationTests/CommsRecoveryTests.cs @@ -87,26 +87,26 @@ public sealed class CommsRecoveryTests(ModbusSimulatorFixture sim) } /// - /// U1 (DisableFC23 wired): a multi-register write on the default (FC23 write + read-back) path + /// U1 (FC23 opt-in, DisableFC23 kill-switch): a multi-register write on the opted-in FC23 write + read-back path /// round-trips against the real server — whether it implements FC23 or answers Illegal Function and - /// is moved to FC16 — and so does the forced-FC16 path. Uses the standard profile's scratch HR[200..209]. + /// is moved to FC16 — and so does the default FC16 path. Uses the standard profile's scratch HR[200..209]. /// [Theory] - [InlineData(false, 200)] - [InlineData(true, 202)] - public async Task Multi_register_write_round_trips_with_and_without_FC23(bool disableFc23, int address) + [InlineData(true, 200)] + [InlineData(false, 202)] + public async Task Multi_register_write_round_trips_with_and_without_FC23(bool useFc23, int address) { SkipUnlessStandardLive(); var tag = new ModbusTagDefinition("F", ModbusRegion.HoldingRegisters, (ushort)address, ModbusDataType.Float32); var opts = new ModbusDriverOptions { - Host = sim.Host, Port = sim.Port, UnitId = 1, Timeout = TimeSpan.FromSeconds(2), DisableFC23 = disableFc23, + Host = sim.Host, Port = sim.Port, UnitId = 1, Timeout = TimeSpan.FromSeconds(2), UseFC23ForMultiRegisterWrites = useFc23, RawTags = ModbusRawTags.Entries([tag]), Probe = new ModbusProbeOptions { Enabled = false }, }; await using var driver = new ModbusDriver(opts, "modbus-u1"); await driver.InitializeAsync("{}", TestContext.Current.CancellationToken); - var value = disableFc23 ? 12.25f : 6.5f; + var value = useFc23 ? 6.5f : 12.25f; for (var i = 0; i < 2; i++) // second write exercises the latched path when the server lacks FC23 { var w = await driver.WriteAsync([new WriteRequest("F", value + i)], TestContext.Current.CancellationToken); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs index 45940c37..14c058c4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingDefaultsTests.cs @@ -49,6 +49,32 @@ public sealed class ModbusCoalescingDefaultsTests r.Select(s => (short)s.Value!).ShouldBe(Enumerable.Range(1, 10).Select(i => (short)i)); } + /// + /// Fix round 1 (coordinator ruling 2): an explicitly authored maxReadGap 0 — what the pre-#741 form + /// persisted on every driver — keeps its old meaning, NO coalescing; the gap-8 default applies only to an + /// absent key (positive control). + /// + [Theory] + [InlineData("""{"host":"h","maxReadGap":0,"disableFC23":false}""", 2)] + [InlineData("""{"host":"h"}""", 1)] + public async Task A_persisted_maxReadGap_0_keeps_coalescing_off(string legacyConfig, int expectedReads) + { + var parsed = ModbusDriverFactoryExtensions.ParseOptions("m1", legacyConfig); + var tags = new[] + { + new ModbusTagDefinition("A", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16), + new ModbusTagDefinition("B", ModbusRegion.HoldingRegisters, 101, ModbusDataType.Int16), + }; + var (drv, t) = await StartAsync(new ModbusDriverOptions + { + Host = "f", MaxReadGap = parsed.MaxReadGap, DisableReadCoalescing = parsed.DisableReadCoalescing, + RawTags = ModbusRawTags.Entries(tags), Probe = new ModbusProbeOptions { Enabled = false }, + }); + await using var _ = drv; + await drv.ReadAsync(["A", "B"], TestContext.Current.CancellationToken); + t.Requests(0x03).Count.ShouldBe(expectedReads); + } + [Fact] public async Task DisableReadCoalescing_reads_every_tag_alone() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs index 72542014..187176cd 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusCoalescingTests.cs @@ -42,9 +42,9 @@ public sealed class ModbusCoalescingTests public ValueTask DisposeAsync() => ValueTask.CompletedTask; } - /// MaxReadGap=0 coalesces only adjacent tags: a one-register hole splits the read. + /// MaxReadGap=0 means no coalescing (its historical meaning, kept for persisted configs). [Fact] - public async Task MaxReadGap_Zero_Does_Not_Bridge_A_Gap() + public async Task MaxReadGap_Zero_Means_No_Coalescing() { var fake = new CountingTransport(); var t1 = new ModbusTagDefinition("T1", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16); @@ -56,7 +56,7 @@ public sealed class ModbusCoalescingTests await drv.ReadAsync(["T1", "T2"], CancellationToken.None); - // A one-register gap with MaxReadGap=0: expect 2 separate FC03 reads. + // MaxReadGap=0 is coalescing OFF: expect 2 separate FC03 reads. var fc03Reads = fake.Reads.Where(r => r.Fc == 0x03).ToList(); fc03Reads.Count.ShouldBe(2); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs index 6ed8e509..4fc4f9b6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs @@ -39,12 +39,12 @@ public sealed class ModbusFamilyWiringTests } [Fact] - public void The_address_string_carries_type_order_and_count_and_keeps_the_other_keys() + public void A_string_only_blob_carries_type_order_and_count_and_keeps_the_other_keys() { ModbusTagDefinitionFactory.FromTagConfig( - """{"addressString":"40011:F:CDAB:4","writable":false,"unitId":3,"region":"Coils","address":9}""", + """{"addressString":"40011:F:CDAB:4","writable":false,"unitId":3}""", "T", ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); - def.Region.ShouldBe(ModbusRegion.HoldingRegisters, "addressString is authoritative over stale resolved keys"); + def.Region.ShouldBe(ModbusRegion.HoldingRegisters); def.Address.ShouldBe((ushort)10); def.DataType.ShouldBe(ModbusDataType.Float32); def.ByteOrder.ShouldBe(ModbusByteOrder.WordSwap); @@ -53,6 +53,22 @@ public sealed class ModbusFamilyWiringTests def.UnitId.ShouldBe((byte)3); } + /// + /// Fix round 1 (review m2): persisted region/address are authoritative over an addressString, so a + /// later Family / MelsecSubFamily change can never silently remap a live tag the editor resolved. + /// + [Fact] + public void Persisted_resolved_keys_win_over_the_address_string_so_a_family_change_cannot_remap() + { + // The editor resolved D100 under MELSEC and persisted HR 100; the driver is later switched to DL205. + const string json = """{"addressString":"D100","region":"HoldingRegisters","address":100,"dataType":"Int16"}"""; + foreach (var family in new[] { ModbusFamily.MELSEC, ModbusFamily.DL205, ModbusFamily.Generic }) + { + ModbusTagDefinitionFactory.FromTagConfig(json, "T", family, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); + def.Address.ShouldBe((ushort)100, $"family {family}"); + } + } + [Fact] public void An_unparseable_address_string_rejects_the_tag_and_the_deploy_gate_says_why() { @@ -83,31 +99,52 @@ public sealed class ModbusFamilyWiringTests [Fact] public void Family_and_MelsecSubFamily_parse_from_the_driver_config() { - var o = ModbusDriverFactoryExtensions.ParseOptions("m1", """{"host":"h","family":"MELSEC","melsecSubFamily":"F_iQF","disableFC23":true}"""); + var o = ModbusDriverFactoryExtensions.ParseOptions("m1", + """{"host":"h","family":"MELSEC","melsecSubFamily":"F_iQF","disableFC23":true,"useFC23ForMultiRegisterWrites":true}"""); o.Family.ShouldBe(ModbusFamily.MELSEC); o.MelsecSubFamily.ShouldBe(MelsecFamily.F_iQF); o.DisableFC23.ShouldBeTrue(); + o.UseFC23ForMultiRegisterWrites.ShouldBeTrue(); } - // ---------------- DisableFC23 gates the FC23 multi-register write ---------------- + // ---------------- FC23 is OPT-IN; DisableFC23 is the kill-switch ---------------- - private static async Task<(ModbusDriver, ScriptedTransport)> StartWriteAsync(bool disableFc23) + private static async Task<(ModbusDriver, ScriptedTransport)> StartWriteAsync(string driverConfigJson) { var t = new ScriptedTransport(); - var opts = new ModbusDriverOptions - { - Host = "f", DisableFC23 = disableFc23, Probe = new ModbusProbeOptions { Enabled = false }, - RawTags = ModbusRawTags.Entries([new ModbusTagDefinition("F", ModbusRegion.HoldingRegisters, 20, ModbusDataType.Float32)]), - }; + var opts = ModbusDriverFactoryExtensions.ParseOptions("m1", driverConfigJson) is var parsed + ? new ModbusDriverOptions + { + Host = "f", DisableFC23 = parsed.DisableFC23, + UseFC23ForMultiRegisterWrites = parsed.UseFC23ForMultiRegisterWrites, + Probe = new ModbusProbeOptions { Enabled = false }, + RawTags = ModbusRawTags.Entries([new ModbusTagDefinition("F", ModbusRegion.HoldingRegisters, 20, ModbusDataType.Float32)]), + } + : null!; var drv = new ModbusDriver(opts, "m1", _ => t); await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); return (drv, t); } - [Fact] - public async Task Multi_register_writes_use_FC23_write_and_read_back_by_default() + /// + /// Fix round 1 (coordinator ruling 1): a config without the opt-in key keeps the FC16 wire — including the + /// legacy shape every form-saved config carries ("disableFC23": false). + /// + [Theory] + [InlineData("""{"host":"h"}""")] + [InlineData("""{"host":"h","disableFC23":false,"maxReadGap":0,"useFC15ForSingleCoilWrites":false}""")] + public async Task Without_the_opt_in_multi_register_writes_stay_on_FC16(string legacyConfig) { - var (drv, t) = await StartWriteAsync(disableFc23: false); + var (drv, t) = await StartWriteAsync(legacyConfig); + await using var _ = drv; + (await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); + t.Sent.Select(s => s.Pdu[0]).ShouldBe([(byte)0x10]); + } + + [Fact] + public async Task Opted_in_multi_register_writes_use_FC23_write_and_read_back() + { + var (drv, t) = await StartWriteAsync("""{"host":"h","useFC23ForMultiRegisterWrites":true}"""); await using var _ = drv; var w = await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken); w[0].StatusCode.ShouldBe(0u); @@ -116,18 +153,18 @@ public sealed class ModbusFamilyWiringTests } [Fact] - public async Task DisableFC23_writes_with_FC16() + public async Task DisableFC23_overrides_the_opt_in() { - var (drv, t) = await StartWriteAsync(disableFc23: true); + var (drv, t) = await StartWriteAsync("""{"host":"h","useFC23ForMultiRegisterWrites":true,"disableFC23":true}"""); await using var _ = drv; (await drv.WriteAsync([new WriteRequest("F", 1.5f)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); t.Sent.Select(s => s.Pdu[0]).ShouldBe([(byte)0x10]); } [Fact] - public async Task A_device_without_FC23_falls_back_to_FC16_once_and_stays_on_it() + public async Task An_opted_in_device_without_FC23_falls_back_to_FC16_once_and_stays_on_it() { - var (drv, t) = await StartWriteAsync(disableFc23: false); + var (drv, t) = await StartWriteAsync("""{"host":"h","useFC23ForMultiRegisterWrites":true}"""); await using var _ = drv; t.Handler = (_, pdu, _) => pdu[0] == 0x17 ? Task.FromException(new ModbusException(0x17, 0x01, "illegal function")) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs index 0e591189..8d90d7f7 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusHealthContractTests.cs @@ -156,8 +156,10 @@ public sealed class ModbusHealthContractTests // ---------------- H7: write status mapping + write-side health ---------------- [Fact] - public async Task Write_transport_failure_is_BadCommunicationError_and_degrades_the_write_side() + public async Task Write_transport_failure_is_BadCommunicationError_and_a_good_read_clears_it() { + // Fix round 1 (review Important #3): ONE health writer for reads and writes. A single write blip used + // to latch Degraded until the next good WRITE — forever, in a read-mostly plant. var (drv, t) = await StartAsync(Opts(Hr("A", 10))); await using var _ = drv; t.Handler = (_, pdu, _) => pdu[0] == 0x06 ? Task.FromException(new IOException("dropped")) : null; @@ -166,16 +168,49 @@ public sealed class ModbusHealthContractTests w[0].StatusCode.ShouldBe(BadCommunicationError); drv.GetHealth().State.ShouldBe(DriverState.Degraded); - // A good READ does not clear a write-side fault... (await drv.ReadAsync(["A"], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(Good); + drv.GetHealth().State.ShouldBe(DriverState.Healthy, "a good read proves the shared transport answered"); + } + + [Fact] + public async Task A_good_write_clears_a_read_side_comms_fault() + { + var (drv, t) = await StartAsync(Opts(Hr("A", 10))); + await using var _ = drv; + t.Handler = (_, pdu, _) => pdu[0] == 0x03 ? Task.FromException(new IOException("dropped")) : null; + await drv.ReadAsync(["A"], TestContext.Current.CancellationToken); drv.GetHealth().State.ShouldBe(DriverState.Degraded); - // ...a good WRITE does. - t.Handler = null; (await drv.WriteAsync([new WriteRequest("A", (short)2)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(Good); drv.GetHealth().State.ShouldBe(DriverState.Healthy); } + [Fact] + public async Task A_malformed_reply_does_not_fail_the_rest_of_the_unit_fast() + { + // Review m3: a byte-count mismatch is a reply that ARRIVED — only a timeout / desync / IO fails the unit fast. + var (drv, t) = await StartAsync(Opts(Hr("Bad", 10), Hr("Fine", 50)) is var o + ? new ModbusDriverOptions { Host = o.Host, RawTags = o.RawTags, Probe = o.Probe, DisableReadCoalescing = true } + : null!); + await using var _ = drv; + t.Bank.HoldingRegisters[50] = 9; + t.Handler = (_, pdu, _) => Addr(pdu) == 10 ? Task.FromResult(new byte[] { 0x03, 0x04, 0, 0 }) : null; + + var r = await drv.ReadAsync(["Bad", "Fine"], TestContext.Current.CancellationToken); + r[0].StatusCode.ShouldBe(BadCommunicationError); + r[1].Value.ShouldBe((short)9, "the sibling on the same unit is still read"); + } + + [Fact] + public async Task DisposeAsync_leaves_health_Unknown() + { + // Review m1: Dispose bumps the generation (dropping a concurrent shutdown's write), so it must write its own. + var (drv, _) = await StartAsync(Opts(Hr("A", 10))); + drv.GetHealth().State.ShouldBe(DriverState.Healthy); + await drv.DisposeAsync(); + drv.GetHealth().State.ShouldBe(DriverState.Unknown); + } + [Fact] public async Task A_write_refusal_counts_as_answered_and_clears_a_write_side_fault() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs index b4acdaee..f7139789 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusRegionTypeValidationTests.cs @@ -14,14 +14,47 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; public sealed class ModbusRegionTypeValidationTests { [Theory] - [InlineData("""{"region":"Coils","address":1,"dataType":"Int16"}""")] - [InlineData("""{"region":"DiscreteInputs","address":1,"dataType":"Float32"}""")] [InlineData("""{"region":"HoldingRegisters","address":1,"dataType":"Bool"}""")] [InlineData("""{"region":"InputRegisters","address":1,"dataType":"Bool"}""")] - public void Incompatible_region_and_type_is_rejected(string json) + public void A_register_region_with_Bool_is_rejected(string json) { + // That pair never read (RegisterCount threw), so refusing it breaks nothing that worked. ModbusTagDefinitionFactory.FromTagConfig(json, "T", out _).ShouldBeFalse(); - ModbusTagDefinitionFactory.Inspect(json).ShouldContain(w => w.Contains("only") || w.Contains("register")); + ModbusTagDefinitionFactory.Inspect(json).ShouldContain(w => w.Contains("register")); + } + + /// + /// Fix round 1 (review Important #2): the pre-#741 editor defaulted dataType to Int16 and persisted it for + /// coil / DI tags, and the old driver ignored it for bit regions — those tags worked and must keep loading. + /// A bit region with any type is coerced to Bool and draws no deploy warning (a warning can be promoted to + /// a blocking error by TagConfigValidationMode.Error, which would refuse an existing config). + /// + [Theory] + [InlineData("""{"region":"Coils","address":3,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0}""")] + [InlineData("""{"region":"DiscreteInputs","address":1,"dataType":"Float32"}""")] + public void A_legacy_bit_region_tag_with_a_register_type_still_loads_as_Bool(string legacyJson) + { + ModbusTagDefinitionFactory.FromTagConfig(legacyJson, "T", out var def).ShouldBeTrue(); + def.DataType.ShouldBe(ModbusDataType.Bool); + ModbusTagDefinitionFactory.Inspect(legacyJson).ShouldBeEmpty(); + } + + [Fact] + public async Task A_legacy_Int16_coil_tag_reads_and_writes_the_coil() + { + var fake = new ModbusDriverTests.FakeTransport(); + fake.Coils[3] = true; + var opts = new ModbusDriverOptions + { + Host = "f", Probe = new ModbusProbeOptions { Enabled = false }, + RawTags = [new RawTagEntry("Run", """{"region":"Coils","address":3,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0}""", false)], + }; + await using var drv = new ModbusDriver(opts, "m1", _ => fake); + await drv.InitializeAsync("{}", TestContext.Current.CancellationToken); + + (await drv.ReadAsync(["Run"], TestContext.Current.CancellationToken))[0].Value.ShouldBe(true); + (await drv.WriteAsync([new WriteRequest("Run", false)], TestContext.Current.CancellationToken))[0].StatusCode.ShouldBe(0u); + fake.Coils[3].ShouldBeFalse(); } [Theory] diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs index 49abe9ab..c868b2bf 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs @@ -137,10 +137,24 @@ public sealed class ModbusTagConfigModelTests ModbusTagConfigModel.FromJson(m.ToJson()).Validate().ShouldBeNull(); } - /// A Coils + Int16 blob (CSV / API) is refused at authoring, matching the driver. + /// A register region with Bool (CSV / API) is refused at authoring, matching the driver. [Fact] - public void Validate_refuses_an_incompatible_region_and_type() - => ModbusTagConfigModel.FromJson("""{"region":"Coils","address":1,"dataType":"Int16"}""").Validate().ShouldNotBeNull(); + public void Validate_refuses_a_register_region_with_Bool() + => ModbusTagConfigModel.FromJson("""{"region":"HoldingRegisters","address":1,"dataType":"Bool"}""").Validate().ShouldNotBeNull(); + + /// + /// Fix round 1: the exact blob the pre-#741 editor persisted for a coil (dataType defaulted to Int16) + /// still validates, loads as Bool (what the driver polls), and re-saves as Bool. + /// + [Fact] + public void A_legacy_Int16_coil_blob_validates_and_loads_as_Bool() + { + const string legacy = """{"region":"Coils","address":3,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0}"""; + var m = ModbusTagConfigModel.FromJson(legacy); + m.Validate().ShouldBeNull(); + m.DataType.ShouldBe(ModbusDataType.Bool); + m.ToJson().ShouldContain("\"dataType\":\"Bool\""); + } /// Review 07 U1: the editor resolves an address string with the DRIVER's family and persists /// both the string and the resolved fields; the driver re-resolves the string with its family. From 0a5aa9fc43bed85fbc4638c83e787991fd1fc9bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:57:13 -0400 Subject: [PATCH 20/34] fix(sql): row-cap key-value and where-pair plans with a TOP/LIMIT N+1 dialect seam (R06:S-1) Key-value and where-pair plans had no row limit, so a selector pointed at a history or events table materialised every matching row on every poll. ISqlDialect gains RowLimitPrefix/RowLimitSuffix(n) (SQL Server TOP (n), Postgres LIMIT n, Oracle FETCH FIRST n ROWS ONLY); plans carry a RowCap (distinct keys, or 1) and ask for RowCap+1 rows; the reader also stops client-side at RowCap+1. Getting the extra row proves the selector ambiguous: the group publishes BadConfigurationError with a rate-limited warning. The reader reports those answered refusals so the driver counts them as answer evidence, and any refusal-only poll now restores Healthy after a comms fault. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Sql.md | 11 +++ .../ISqlDialect.cs | 25 +++++ .../OracleDialect.cs | 15 +++ .../PostgresDialect.cs | 14 +++ .../SqlServerDialect.cs | 16 ++++ .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 31 ++++--- .../SqlGroupPlanner.cs | 18 +++- .../SqlPollReader.cs | 93 +++++++++++++++---- .../SqlQueryPlan.cs | 10 +- .../OracleFlavouredSqliteDialect.cs | 4 + .../ColonMarkerDialect.cs | 4 + .../OracleDialectTests.cs | 9 ++ .../PostgresDialectTests.cs | 9 ++ .../SqlCatalogDefaultSchemaTests.cs | 4 + .../SqlCatalogGateDriverTests.cs | 4 + .../SqlDriverTests.cs | 37 +++++++- .../SqlGroupPlannerChunkingTests.cs | 6 +- .../SqlGroupPlannerTests.cs | 48 ++++++++-- .../SqlPollReaderTests.cs | 93 ++++++++++++++++++- .../SqlServerDialectTests.cs | 9 ++ .../SqliteDialect.cs | 14 +++ 21 files changed, 420 insertions(+), 54 deletions(-) diff --git a/docs/drivers/Sql.md b/docs/drivers/Sql.md index 7b7ad3ae..ffa68922 100644 --- a/docs/drivers/Sql.md +++ b/docs/drivers/Sql.md @@ -166,6 +166,17 @@ emits a `SqlQueryPlan` per distinct group key. Grouping is deterministic (groups order of their first member, members keep their input order, parameters keep first-appearance order), so an unchanged configuration produces byte-identical SQL. +**Every key-value and where-pair query is row-capped (R06:S-1).** A key-value plan asks for one row more +than it has distinct keys (`TOP (n+1)` / `LIMIT n+1` / `FETCH FIRST n+1 ROWS ONLY`), a where-pair wide-row plan +for two, and the reader stops reading there even if a provider ignores the limit. Receiving the extra row +proves the selector is ambiguous — a history/events table, or a where column that is not unique — so every +member of that group publishes `BadConfigurationError` with a rate-limited warning naming the table and +selector, instead of the old behaviour of materialising every matching row on every poll and publishing an +arbitrary "last row" at Good. A duplicate that fits inside the cap (a duplicated key alongside an absent one) +still resolves last-row-wins. `topByTimestamp` keeps its single-row limit. An over-cap answer, like every +other refusal (absent row, NULL cell, uncoercible cell), is the database answering: it advances +`LastSuccessfulRead` and restores `Healthy` after a comms fault. + ⚠️ **Group keys are case-sensitive on identifiers.** Two tags authored `dbo.T` and `DBO.T` plan as **two** groups — one extra round-trip, not a correctness problem, but worth knowing if a group count looks higher than expected. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs index c144ec62..e1069372 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs @@ -177,6 +177,31 @@ public interface ISqlDialect ///
string SingleRowLimitSuffix { get; } + /// + /// The position's fragment for an -row limit — + /// T-SQL's "TOP (n) "; empty for a dialect that limits at the end. Same trailing-space rule. + /// + /// + /// Added for the key-value / where-pair row cap (R06:S-1): those plans used to carry no limit, so an + /// ambiguous selector against a history table materialised every matching row on every poll. The + /// planner asks for N + 1 rows — one more than the plan can use — so ambiguity stays observable + /// (the reader sees the extra row and publishes a configuration error) without unbounded reads. + /// + /// The row limit; at least 1. + /// The fragment, possibly empty. + /// is below 1. + string RowLimitPrefix(int rows); + + /// + /// The position's fragment for an -row limit — + /// " LIMIT n" / " FETCH FIRST n ROWS ONLY"; empty for a dialect that limits after + /// SELECT. Same leading-space rule. See . + /// + /// The row limit; at least 1. + /// The fragment, possibly empty. + /// is below 1. + string RowLimitSuffix(int rows); + /// Catalog query listing schemas — the browser's RootAsync level. Takes no parameters. string ListSchemasSql { get; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs index 2dc72a83..1789959f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs @@ -124,6 +124,21 @@ public sealed class OracleDialect : ISqlDialect // statement shape, which this seam deliberately leaves with the planner. public string SingleRowLimitSuffix => " FETCH FIRST 1 ROWS ONLY"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + // The same 12c+ row-limiting clause as SingleRowLimitSuffix; valid after a WHERE with no ORDER BY. + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " FETCH FIRST " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture) + " ROWS ONLY"; + } + /// // Schemas visible to the connecting principal, from ALL_USERS (Oracle equates a schema with a // user), aliased to the TABLE_SCHEMA column name the catalog readers project by. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs index b332d912..6809b328 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs @@ -86,6 +86,20 @@ public sealed class PostgresDialect : ISqlDialect /// public string SingleRowLimitSuffix => " LIMIT 1"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " LIMIT " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + // The schemas holding relations this connection can actually see, system schemas excluded. // // Sourced from information_schema.tables, not information_schema.schemata - a deliberate diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs index 9db9848c..61057407 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs @@ -87,6 +87,22 @@ public sealed class SqlServerDialect : ISqlDialect /// public string SingleRowLimitSuffix => string.Empty; + /// + // Parenthesised: TOP (n) is the documented modern form, and TOP n without parentheses is the + // backwards-compatibility spelling. + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return "TOP (" + rows.ToString(System.Globalization.CultureInfo.InvariantCulture) + ") "; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + /// public string ListSchemasSql => "SELECT DISTINCT TABLE_SCHEMA FROM INFORMATION_SCHEMA.TABLES ORDER BY TABLE_SCHEMA"; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 945fc32b..4a8936c9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -519,9 +519,9 @@ public sealed class SqlDriver try { - var snapshots = await _reader.ReadAsync(fullReferences, cancellationToken).ConfigureAwait(false); - ObservePollOutcome(snapshots); - return snapshots; + var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); + ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups); + return result.Snapshots; } catch (OperationCanceledException) { @@ -608,7 +608,13 @@ public sealed class SqlDriver /// cadence rather than backed off from. /// /// The snapshots the reader returned for one poll. - internal void ObservePollOutcome(IReadOnlyList snapshots) + /// + /// Groups the database answered but the driver refused (an over-cap result set, R06:S-1). Their + /// members carry , which alone is not answer evidence + /// (the planner mints it before any I/O), so the count is what makes them count as the database + /// answering. + /// + internal void ObservePollOutcome(IReadOnlyList snapshots, int answeredRefusals = 0) { if (snapshots.Count == 0) return; @@ -625,6 +631,8 @@ public sealed class SqlDriver if (IsAnswerEvidence(snapshot.StatusCode)) answered++; } + answered += answeredRefusals; + if (unreachable > 0) { Degrade( @@ -660,17 +668,12 @@ public sealed class SqlDriver if (answered == 0) return; // authoring-class Bad only — not a statement about the database. - if (good == 0) - { - // The database answered, just not with a value: an absent row, a NULL cell, a cell that would - // not coerce. Those are data/authoring facts, so the health STATE is deliberately left exactly - // as it was — but they are still proof the transport works, and the evidence clock the - // comms-loss watchdog reads must say so. - TouchLastSuccessfulRead(); - TransitionHostTo(HostState.Running); - return; - } + // Good values, OR a database that answered with refusals only — an absent row, a NULL cell, an + // uncoercible cell, an over-cap result set. A refusal is the database answering: it advances the + // evidence clock AND restores Healthy after a comms fault, exactly as a Good read does. Leaving the + // state as it was latched a recovered database Degraded behind every poll that read only refusals. + // // Route through the Faulted guard, not a raw WriteHealth: a poll still in flight when a concurrent // ReinitializeAsync has recorded Faulted (the engine's ~5 s teardown wait is shorter than the 15 s // default operationTimeout, so a stale poll can still complete afterwards) must not flip Faulted back diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs index 9d1142b8..31c77351 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs @@ -230,15 +230,21 @@ public static class SqlGroupPlanner values.Add(keyValue); } + // Row cap (R06:S-1): one row per distinct key is all this plan can use, so it asks for ONE more — enough + // to see a source breaking the one-row-per-key contract, never enough to pull a history table's every + // matching row into memory on every poll. + var rowCap = values.Count; var sql = new StringBuilder("SELECT ") + .Append(dialect.RowLimitPrefix(rowCap + 1)) .Append(QuoteList(selected, dialect)) .Append(" FROM ").Append(QuoteQualifiedName(first.Table, dialect)) .Append(" WHERE ").Append(dialect.QuoteIdentifier(keyColumn)) .Append(" IN (").AppendJoin(", ", markers).Append(')') + .Append(dialect.RowLimitSuffix(rowCap + 1)) .ToString(); return new SqlQueryPlan( - SqlTagModel.KeyValue, groupKey, sql, markers, bindNames, values, members, selected); + SqlTagModel.KeyValue, groupKey, sql, markers, bindNames, values, members, selected, rowCap); } /// @@ -270,14 +276,18 @@ public static class SqlGroupPlanner // A where-pair wins over topByTimestamp; the parser already guarantees exactly one is populated. if (!Blank(first.RowSelectorColumn) && first.RowSelectorValue is not null) { + // Capped at TWO rows (R06:S-1): the selector must identify one row, and the second is the proof it + // does not. A TOP 1 alone would hide that ambiguity (see SqlPollReader.SelectWideRow); no cap at all + // materialised every matching row of an events table on every poll. var marker = dialect.ParameterMarker(RowSelectorBaseName); var sql = string.Concat( - "SELECT ", columns, " FROM ", table, - " WHERE ", dialect.QuoteIdentifier(first.RowSelectorColumn!), " = ", marker); + "SELECT ", dialect.RowLimitPrefix(2), columns, " FROM ", table, + " WHERE ", dialect.QuoteIdentifier(first.RowSelectorColumn!), " = ", marker, + dialect.RowLimitSuffix(2)); return new SqlQueryPlan( SqlTagModel.WideRow, groupKey, sql, [marker], [dialect.ParameterBindName(RowSelectorBaseName)], - [first.RowSelectorValue], members, selected); + [first.RowSelectorValue], members, selected, RowCap: 1); } if (!Blank(first.RowSelectorTopByTimestamp)) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index da6ecfb5..25e70a39 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -162,11 +162,24 @@ public sealed class SqlPollReader : IDisposable /// The database could not be reached (opening a connection failed). public async Task> ReadAsync( IReadOnlyList fullReferences, CancellationToken cancellationToken) + => (await ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false)).Snapshots; + + /// + /// plus what the snapshots alone cannot say: how many groups the database + /// ANSWERED with more rows than the plan can use (R06:S-1). Those groups publish + /// — the same code an un-plannable definition gets + /// before any I/O — so the driver needs this count to treat them as the answer evidence they are. + /// + /// The RawPaths to read. + /// The caller's token; see . + /// The snapshots and the answered-refusal count. + public async Task ReadDetailedAsync( + IReadOnlyList fullReferences, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(fullReferences); var results = new DataValueSnapshot[fullReferences.Count]; - if (results.Length == 0) return results; + if (results.Length == 0) return new SqlPollResult(results, 0); var readAt = DateTime.UtcNow; @@ -189,19 +202,23 @@ public sealed class SqlPollReader : IDisposable positions.Add(i); } - if (definitions.Count > 0) - await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false); + var rowCapExceeded = definitions.Count > 0 + ? await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false) + : 0; // Fail-safe. Nothing above should leave a hole, but "N in, N out" is a contract the caller indexes // blind — a null here would be a NullReferenceException in the publish fan-out, far from its cause. for (var i = 0; i < results.Length; i++) results[i] ??= new DataValueSnapshot(null, SqlStatusCodes.BadInternalError, null, readAt); - return results; + return new SqlPollResult(results, rowCapExceeded); } - /// Plans the resolved definitions, runs every group concurrently, and slices the results back. - private async Task ReadGroupsAsync( + /// + /// Plans the resolved definitions, runs every group concurrently, and slices the results back. + /// Returns how many groups came back over their row cap. + /// + private async Task ReadGroupsAsync( List definitions, Dictionary> slots, DataValueSnapshot[] results, @@ -227,7 +244,7 @@ public sealed class SqlPollReader : IDisposable foreach (var position in positions) results[position] = snapshot; } - return; + return 0; } // Groups run concurrently; _gate — not the task count — is what bounds open connections. Each group @@ -241,8 +258,14 @@ public sealed class SqlPollReader : IDisposable // leaves a sibling group running against a connection nobody is watching. var outcomes = await Task.WhenAll(groups).ConfigureAwait(false); + var rowCapExceeded = 0; for (var g = 0; g < plans.Count; g++) + { Slice(plans[g], outcomes[g], slots, results, readAt); + if (ReferenceEquals(outcomes[g], GroupResult.RowCapExceeded)) rowCapExceeded++; + } + + return rowCapExceeded; } /// @@ -502,8 +525,11 @@ public sealed class SqlPollReader : IDisposable columnIndex[selected[c]] = c; } + // The client-side half of the row cap (R06:S-1): the statement already asks for RowCap + 1 rows, but a + // provider that cannot spell a limit must not turn that into an unbounded read either. + var limit = plan.RowCap is { } cap ? cap + 1 : int.MaxValue; var rows = new List(); - while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + while (rows.Count < limit && await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { var row = new object?[selected.Count]; for (var c = 0; c < selected.Count; c++) @@ -511,6 +537,25 @@ public sealed class SqlPollReader : IDisposable rows.Add(row); } + if (plan.RowCap is { } rowCap && rows.Count > rowCap) + { + if (ShouldWarnAboutContract()) + { + var first = plan.Members[0]; + var selector = plan.Model == SqlTagModel.KeyValue + ? string.Concat(first.KeyColumn, " IN (", rowCap.ToString(CultureInfo.InvariantCulture), " key(s))") + : string.Concat(first.RowSelectorColumn, " = ", first.RowSelectorValue); + _logger.LogWarning( + "Sql poll source '{Table}' returned more rows than the {Model} selector '{Selector}' can use " + + "(at most {Cap}) in one poll, so the selector is ambiguous — a history/events table, or a " + + "where column that is not unique. Its {Members} tag(s) publish BadConfigurationError and " + + "nothing past row {Limit} is read. Point the tag at a current-values table or a latest-row view.", + first.Table, plan.Model, selector, rowCap, plan.Members.Count, rowCap + 1); + } + + return GroupResult.RowCapExceeded; + } + return GroupResult.Ok(rows, columnIndex, typeNames); } @@ -609,15 +654,14 @@ public sealed class SqlPollReader : IDisposable /// emits no ORDER BY, so across polls the "last" row is whatever the server returned last, /// i.e. storage order. That is why more than one row is a reported contract violation and not merely /// a tie to break — the model's premise (design §5.3) is that the selector identifies one row. - /// Why the where-pair form is deliberately NOT given the dialect's single-row limit. A - /// TOP 1 with no ORDER BY is not deterministic either — it returns whichever row the - /// plan happens to produce first, so it would trade one arbitrary row for a differently arbitrary - /// one. Worse, it would destroy the only evidence this method has that the selector is ambiguous - /// (Rows.Count > 1), converting a loud misconfiguration into a permanently silent one — - /// precisely the failure mode this class's summary calls its top risk. The topByTimestamp - /// form keeps its row limit because there the ORDER BY at least fixes which rows can - /// be first. The accepted cost is that an ambiguous selector materialises every matching row for one - /// poll; the warning is what gets it fixed. + /// Why the where-pair form is capped at TWO rows, not one (R06:S-1). A TOP 1 with no + /// ORDER BY is not deterministic — it returns whichever row the plan produces first — and it would + /// destroy the only evidence that the selector is ambiguous, converting a loud misconfiguration into a + /// permanently silent one. No limit at all materialised every matching row of an events table on every + /// poll. Two rows keeps the evidence and bounds the read: a second row makes the group publish + /// BadConfigurationError in MaterialiseAsync, so a where-pair group never reaches this + /// method with more than one row. The multi-row warning below is now reached only by a plan with no + /// cap. /// ⚠ The topByTimestamp answer is only defined when the ordering column is unique /// ORDER BY ts DESC plus a one-row limit over a column with ties returns an /// arbitrary row among the tied ones, and — unlike the where-pair form — this method can never @@ -807,6 +851,13 @@ public sealed class SqlPollReader : IDisposable public static GroupResult Failed { get; } = new(SqlStatusCodes.BadCommunicationError, NoRows, NoColumns, []); + /// + /// The database answered with more rows than the plan can use — an ambiguous selector (R06:S-1). + /// A configuration fact, not a database fault; the driver counts it as answer evidence. + /// + public static GroupResult RowCapExceeded { get; } = + new(SqlStatusCodes.BadConfigurationError, NoRows, NoColumns, []); + /// The status every member inherits when is false. public uint FailureStatus { get; } @@ -835,6 +886,14 @@ public sealed class SqlPollReader : IDisposable } } +/// One poll's snapshots plus the answer evidence they cannot carry themselves. +/// One snapshot per requested reference, at the same index. +/// +/// Groups the database answered with more rows than their plan can use; their members carry +/// . +/// +public sealed record SqlPollResult(IReadOnlyList Snapshots, int RowCapExceededGroups); + /// /// The OPC UA status codes the Sql driver publishes, and the quality-class predicates its tests read /// them back with. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs index 8e20c989..333a762c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs @@ -55,6 +55,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// The bare (unquoted) column names in 's SELECT list, in order and distinct — the /// reader's map from result-set ordinal to source column. /// +/// +/// The most rows this plan can USE — its distinct key count (key-value) or 1 (a where-pair wide row); +/// when the statement already limits itself (topByTimestamp). +/// asks for RowCap + 1, and the reader stops there: receiving the extra row proves the selector is +/// ambiguous (a history table, a non-unique where column) and the group publishes +/// BadConfigurationError instead of materialising every matching row on every poll (R06:S-1). +/// public sealed record SqlQueryPlan( SqlTagModel Model, object GroupKey, @@ -63,7 +70,8 @@ public sealed record SqlQueryPlan( IReadOnlyList ParameterBindNames, IReadOnlyList Parameters, IReadOnlyList Members, - IReadOnlyList SelectedColumns) + IReadOnlyList SelectedColumns, + int? RowCap = null) { // A plan is documented as safe to cache and reuse across polls, so it must not stay aliased to the // planner's mutable working lists — an IReadOnlyList holding a List is downcastable, and a diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs index a6868aa3..0987f4b8 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs @@ -38,6 +38,10 @@ public sealed class OracleFlavouredSqliteDialect : ISqlDialect /// public string SingleRowLimitSuffix => _inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => _inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => _inner.RowLimitSuffix(rows); + /// public string ListSchemasSql => _inner.ListSchemasSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs index f514b1be..939a267b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs @@ -77,6 +77,10 @@ public sealed class ColonMarkerDialect : ISqlDialect /// public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + /// public string ListSchemasSql => Colonize(Inner.ListSchemasSql); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs index a0050cf7..70629ae4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs @@ -210,6 +210,15 @@ public class OracleDialectTests public void LivenessSql_selectsFromDual() => new OracleDialect().LivenessSql.ShouldBe("SELECT 1 FROM DUAL"); + [Fact] + public void RowLimit_isFetchFirstN_inTheSuffixPosition() + { + var d = new OracleDialect(); + d.RowLimitPrefix(3).ShouldBe(""); + d.RowLimitSuffix(3).ShouldBe(" FETCH FIRST 3 ROWS ONLY"); + Should.Throw(() => d.RowLimitSuffix(0)); + } + [Fact] public void SingleRowLimit_isFetchFirst_inTheSuffixPosition_withItsOwnLeadingSpace() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs index 2f56b7d1..de681931 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs @@ -226,6 +226,15 @@ public class PostgresDialectTests d.ListColumnsSql.ShouldContain("\"IS_NULLABLE\""); } + [Fact] + public void RowLimit_isLimitN_inTheSuffixPosition() + { + var d = new PostgresDialect(); + d.RowLimitPrefix(3).ShouldBe(""); + d.RowLimitSuffix(3).ShouldBe(" LIMIT 3"); + Should.Throw(() => d.RowLimitSuffix(0)); + } + [Fact] public void SingleRowLimit_isLimit1_inTheSuffixPosition_withItsOwnLeadingSpace() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs index 0c72ca4a..2f8a1f47 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs @@ -113,6 +113,10 @@ public sealed class SqlCatalogDefaultSchemaTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql => twoSchemas ? Inner.ListSchemasSql + " UNION ALL SELECT 'other' AS TABLE_SCHEMA" : Inner.ListSchemasSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs index f0e5d373..017c0bd9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs @@ -374,6 +374,10 @@ public sealed class SqlCatalogGateDriverTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql { get; } = listSchemasSql; public string DefaultSchemaSql => Inner.DefaultSchemaSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 492cc05d..6dd21ce9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -272,14 +272,15 @@ public sealed class SqlDriverTests // ---- the comms-loss watchdog's evidence clock (Driver.Sql-006) ---- [Fact] - public async Task ReadAsync_whenTheDatabaseAnswersWithNoRow_advancesTheEvidenceClock_withoutChangingHealth() + public async Task ReadAsync_whenTheDatabaseAnswersWithNoRow_advancesTheEvidenceClock_andRestoresHealthy() { // Driver.Sql-006. LastSuccessfulRead is what DriverInstanceActor's comms-loss watchdog reads: a // non-Healthy driver whose clock has been frozen for 90 s is force-reconnected (Gitea #527), on the // premise that the clock is the honest evidence the transport works. BadNoData IS that evidence — // the query ran and the row is absent — so a driver whose key set was deleted must not be - // reconnected every 90 s against a database answering every poll on time. The health STATE stays - // exactly as it was: an absent row is a data fact, not a connectivity one. + // reconnected every 90 s against a database answering every poll on time. And since the database + // answered, the driver is Healthy again (lane rule: a refusal means the device answered — it used to + // leave the Degraded latched behind every refusal-only poll). using var fixture = new SqlitePollFixture(); await using var driver = NewDriver(fixture, KvEntry("Missing", SqlitePollFixture.AbsentKey)); await driver.InitializeAsync(ConfigJson, CancellationToken.None); @@ -295,7 +296,7 @@ public sealed class SqlDriverTests snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); var health = driver.GetHealth(); health.LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); // RED pre-fix: frozen - health.State.ShouldBe(DriverState.Degraded); // classification untouched + health.State.ShouldBe(DriverState.Healthy); // the database answered } [Fact] @@ -356,6 +357,34 @@ public sealed class SqlDriverTests driver.GetHealth().LastSuccessfulRead.ShouldBe(before); } + /// + /// R06:S-1. An over-cap result set publishes BadConfigurationError — the code the planner also + /// mints before any I/O, which on its own is NOT answer evidence (theory above). The reader's + /// answered-refusal count is what makes it count: the database answered, so the clock advances and a + /// comms-Degraded driver is Healthy again. Control: the same snapshot with no count leaves both alone. + /// + [Fact] + public async Task ObservePollOutcome_aRowCapRefusal_isAnswerEvidence_andRestoresHealthy() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + var refused = new DataValueSnapshot(null, SqlStatusCodes.BadConfigurationError, null, DateTime.UtcNow); + + driver.ObservePollOutcome([new DataValueSnapshot(null, SqlStatusCodes.BadTimeout, null, DateTime.UtcNow)]); + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + var before = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + await Task.Delay(20, TestContext.Current.CancellationToken); + + driver.ObservePollOutcome([refused]); // no count: planner-minted, no evidence + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + driver.GetHealth().LastSuccessfulRead.ShouldBe(before); + + driver.ObservePollOutcome([refused], answeredRefusals: 1); // the database answered, over-cap + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs index d8bc6c8a..dd6920c2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs @@ -150,7 +150,7 @@ public sealed class SqlGroupPlannerChunkingTests plan.ParameterNames.ShouldBe( Enumerable.Range(0, plan.Parameters.Count) .Select(i => "@k" + i.ToString(CultureInfo.InvariantCulture))); - plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); + plan.SqlText.ShouldContain("IN (" + string.Join(", ", plan.ParameterNames) + ")"); } } @@ -297,6 +297,10 @@ public sealed class SqlGroupPlannerChunkingTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql => Inner.ListSchemasSql; public string DefaultSchemaSql => Inner.DefaultSchemaSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs index 344f4447..058ea516 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs @@ -29,6 +29,38 @@ public class SqlGroupPlannerTests RowSelectorColumn: selectorColumn, RowSelectorValue: selectorValue, RowSelectorTopByTimestamp: topByTimestamp); + // ---- R06:S-1: the row cap ---- + + /// + /// A key-value plan asks for one row more than it has distinct keys, and a where-pair wide-row plan for + /// two: enough to SEE an ambiguous selector, never enough to materialise a history table every poll. + /// The topByTimestamp form keeps its single-row limit and needs no cap. + /// + [Fact] + public void Plans_carryARowCap_andAskForOneRowMore() + { + var plans = SqlGroupPlanner.Plan( + [ + KvTag("A", "dbo.TagValues", "tag_name", "k0", "num_value", null), + KvTag("B", "dbo.TagValues", "tag_name", "k1", "num_value", null), + KvTag("C", "dbo.TagValues", "tag_name", "k1", "num_value", null), // same key: one row serves both + WideTag("W", "dbo.LatestStatus", "oven_temp", selectorColumn: "station_id", selectorValue: "7"), + WideTag("T", "dbo.Readings", "oven_temp", topByTimestamp: "sample_ts"), + ], new SqlServerDialect()).ToList(); + + var kv = plans.Single(p => p.Model == SqlTagModel.KeyValue); + kv.RowCap.ShouldBe(2); + kv.SqlText.ShouldStartWith("SELECT TOP (3) "); + + var wherePair = plans.Single(p => p.Parameters.Count == 1 && p.Model == SqlTagModel.WideRow); + wherePair.RowCap.ShouldBe(1); + wherePair.SqlText.ShouldStartWith("SELECT TOP (2) "); + + var top = plans.Single(p => p.Parameters.Count == 0); + top.RowCap.ShouldBeNull(); + top.SqlText.ShouldStartWith("SELECT TOP 1 "); + } + // ---- the plan's golden case ---- [Fact] @@ -58,7 +90,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].ParameterNames.ShouldBe(new[] { "@k0" }); plans[0].SelectedColumns.ShouldBe(new[] { "tag_name", "num_value", "sample_ts" }); plans[0].Model.ShouldBe(SqlTagModel.KeyValue); @@ -75,7 +107,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].TimestampColumn.ShouldBeNull(); } @@ -181,7 +213,7 @@ public class SqlGroupPlannerTests var plan = plans[0]; plan.ParameterNames.Count.ShouldBe(plan.Parameters.Count); - plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); + plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); // T-SQL caps at the front plan.Parameters.ShouldBe(new object[] { "k0", "k1", "k2" }); } @@ -201,7 +233,7 @@ public class SqlGroupPlannerTests KvTag("B", "TagValues", "tag_name", "Line1.Temp", "num_value", null), ], new ColonMarkerDialect()).ToList(); - plans[0].SqlText.ShouldEndWith("IN (:k0, :k1)"); + plans[0].SqlText.ShouldEndWith("IN (:k0, :k1) LIMIT 3"); plans[0].SqlText.ShouldNotContain("@"); plans[0].ParameterNames.ShouldBe(new[] { ":k0", ":k1" }); } @@ -213,7 +245,7 @@ public class SqlGroupPlannerTests [WideTag("A", "LatestStatus", "oven_temp", selectorColumn: "station_id", selectorValue: "7")], new ColonMarkerDialect()).ToList(); - plans[0].SqlText.ShouldEndWith("= :w"); + plans[0].SqlText.ShouldEndWith("= :w LIMIT 2"); plans[0].SqlText.ShouldNotContain("@"); plans[0].ParameterNames.ShouldBe(new[] { ":w" }); } @@ -277,7 +309,7 @@ public class SqlGroupPlannerTests plans.Count.ShouldBe(1); plans[0].SqlText.ShouldBe( - "SELECT [oven_temp], [pressure] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); + "SELECT TOP (2) [oven_temp], [pressure] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); plans[0].ParameterNames.ShouldBe(new[] { "@w" }); plans[0].Parameters.ShouldBe(new object[] { "7" }); // the whereValue is bound, never inlined plans[0].SelectedColumns.ShouldBe(new[] { "oven_temp", "pressure" }); @@ -313,7 +345,7 @@ public class SqlGroupPlannerTests plans.Count.ShouldBe(1); plans[0].SelectedColumns.ShouldBe(new[] { "oven_temp", "pressure", "sample_ts" }); plans[0].SqlText.ShouldBe( - "SELECT [oven_temp], [pressure], [sample_ts] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); + "SELECT TOP (2) [oven_temp], [pressure], [sample_ts] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); } [Fact] @@ -424,7 +456,7 @@ public class SqlGroupPlannerTests // A column name is an identifier — it cannot be parameterized, so the guarantee is that it is // bracket-quoted with ] doubled, leaving one (nonexistent) identifier rather than executable SQL. plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); } // ---- contract edges ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index 87a0d02a..4181449c 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -187,7 +187,7 @@ public sealed class SqlPollReaderTests } [Fact] - public async Task ReadAsync_wideRow_whenTheSelectorMatchesManyRows_warnsInsteadOfSilentlyPickingOne() + public async Task ReadAsync_wideRow_whenTheSelectorMatchesManyRows_publishesAConfigurationError_andWarns() { using var fixture = new SqlitePollFixture(); // A second row for the SAME station. The where-pair form emits no ORDER BY and no row limit, so @@ -201,8 +201,11 @@ public sealed class SqlPollReaderTests var snapshots = await reader.ReadAsync(["Oven"], CancellationToken.None); - // Still a value — the reader degrades to "last row wins", exactly as the key-value model does. - snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + // R06:S-1: the where-pair plan now asks for TWO rows (one more than it can use). Getting the second is + // proof the selector is ambiguous, so the group publishes a configuration error rather than a + // storage-order-dependent "last row" value — and nothing past that second row is ever read. + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[0].Value.ShouldBeNull(); var warning = logger.Entries.ShouldHaveSingleItem(); warning.Level.ShouldBe(LogLevel.Warning); @@ -277,13 +280,93 @@ public sealed class SqlPollReaderTests $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + $"VALUES ('{SqlitePollFixture.PresentKey}', 99.0, '2026-07-24T10:00:09Z')"); - var reader = Reader(fixture, KvTag("Speed", SqlitePollFixture.PresentKey)); + // Two keys, one absent: the duplicate fits inside the row cap (2), so it is not provably a history + // table — the last-row-wins rule still applies there (the cap test below covers the other side). + var reader = Reader(fixture, + KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Gone", SqlitePollFixture.AbsentKey)); - var snapshots = await reader.ReadAsync(["Speed"], CancellationToken.None); + var snapshots = await reader.ReadAsync(["Speed", "Gone"], CancellationToken.None); // Design §3.6: the contract is one row per key; when a source breaks it the reader is at least // deterministic — last occurrence in reader order — rather than silently arbitrary. snapshots[0].Value.ShouldBe(99.0); + snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); + } + + /// + /// R06:S-1. A key-value tag pointed at a history table (many rows per key) used to pull every + /// matching row into memory on every poll. The plan now asks for one row more than it has keys; a full + /// result set proves the selector is ambiguous, so every member publishes a configuration error and a + /// (rate-limited) warning names the table. The positive control: the same tag against a clean source + /// reads Good. + /// + [Fact] + public async Task ReadAsync_keyValueRowCapExceeded_publishesAConfigurationError_forEveryMember() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var clean = Reader(fixture, logger, KvTag("Speed", SqlitePollFixture.PresentKey)); + (await clean.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + logger.Entries.ShouldBeEmpty(); + + for (var i = 0; i < 50; i++) + { + fixture.Execute(string.Create(CultureInfo.InvariantCulture, + $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + + $"VALUES ('{SqlitePollFixture.PresentKey}', {i}, '2026-07-24T11:00:00Z')")); + } + + var reader = Reader(fixture, logger, KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Speed2", SqlitePollFixture.PresentKey)); + var snapshots = await reader.ReadAsync(["Speed", "Speed2"], CancellationToken.None); + + snapshots.ShouldAllBe(x => x.StatusCode == SqlStatusCodes.BadConfigurationError); + var warning = logger.Entries.ShouldHaveSingleItem(); + warning.Level.ShouldBe(LogLevel.Warning); + warning.Message.ShouldContain(SqlitePollFixture.KeyValueTable); + } + + /// + /// The client-side half of the cap: a dialect whose limit fragments are empty (a provider that cannot + /// spell one) must still stop reading at cap + 1 and report the same configuration error. + /// + [Fact] + public async Task ReadAsync_aDialectThatEmitsNoLimit_isStillCappedClientSide() + { + using var fixture = new SqlitePollFixture(); + AddSecondRowForPresentStation(fixture); + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new NoLimitDialect(), + commandTimeout: CommandTimeout, operationTimeout: OperationTimeout, + maxConcurrentGroups: 4, nullIsBad: false, + resolve: Resolver(WideTag("Oven", "oven_temp", selectorValue: SqlitePollFixture.PresentStation))); + + var snapshots = await reader.ReadAsync(["Oven"], CancellationToken.None); + + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + } + + /// SQLite with the row-limit fragments blanked — the "provider cannot spell a limit" shape. + private sealed class NoLimitDialect : ISqlDialect + { + private readonly SqliteDialect _inner = new(); + public SqlProvider Provider => _inner.Provider; + public System.Data.Common.DbProviderFactory Factory => _inner.Factory; + public int MaxBindParameters => _inner.MaxBindParameters; + public string LivenessSql => _inner.LivenessSql; + public string SingleRowLimitPrefix => _inner.SingleRowLimitPrefix; + public string SingleRowLimitSuffix => _inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => string.Empty; + public string RowLimitSuffix(int rows) => string.Empty; + public string ListSchemasSql => _inner.ListSchemasSql; + public string DefaultSchemaSql => _inner.DefaultSchemaSql; + public string ListTablesSql => _inner.ListTablesSql; + public string ListColumnsSql => _inner.ListColumnsSql; + public string QuoteIdentifier(string ident) => _inner.QuoteIdentifier(ident); + public string ParameterMarker(string baseName) => _inner.ParameterMarker(baseName); + public string ParameterBindName(string baseName) => _inner.ParameterBindName(baseName); + public void ConfigureCommand(System.Data.Common.DbCommand command) => _inner.ConfigureCommand(command); + public void ConfigureParameter(System.Data.Common.DbParameter parameter, object? value) => ((ISqlDialect)_inner).ConfigureParameter(parameter, value); + public DriverDataType MapColumnType(string sqlDataType) => _inner.MapColumnType(sqlDataType); } // ---- the frozen-peer contract ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs index 901382f1..912a8ee6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs @@ -13,6 +13,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests; /// public class SqlServerDialectTests { + [Fact] + public void RowLimit_isTopN_inThePrefixPosition() + { + var d = new SqlServerDialect(); + d.RowLimitPrefix(3).ShouldBe("TOP (3) "); + d.RowLimitSuffix(3).ShouldBe(""); + Should.Throw(() => d.RowLimitPrefix(0)); + } + [Fact] public void QuoteIdentifier_bracketsAndEscapesEmbeddedBrackets() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs index 62b9f3f4..20dded60 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs @@ -87,6 +87,20 @@ public sealed class SqliteDialect : ISqlDialect /// public string SingleRowLimitSuffix => " LIMIT 1"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " LIMIT " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + /// /// SQLite has no schema namespace, so the schema level is the single literal /// — enough to give the browser a real root to expand. From ab29b76be0feddc39844e8146230920dc670e543 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:00:25 -0400 Subject: [PATCH 21/34] perf(sql): rate-limit per-poll warnings and cache poll plans per reference list (R06:S-6, S-7) S-6: a frozen group's BadTimeout warning and a failing group's query-error warning (with SQL text) logged every poll per group indefinitely; both are now rate-limited per plan (once a minute), and an uncoercible cell warns once per tag instead of once per poll. Every poll still publishes the Bad status. S-7: plans and the slot map were rebuilt every poll although a poll loop passes the same reference list each cycle. They are cached per list instance (weakly keyed) and rebuilt only when a reference resolves to a different definition instance, which a catalog-gate swap produces. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlPollReader.cs | 183 ++++++++++++++---- .../SqlPollReaderTests.cs | 120 ++++++++++++ 2 files changed, 264 insertions(+), 39 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index 25e70a39..fdb22ff0 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -1,5 +1,7 @@ +using System.Collections.Concurrent; using System.Data.Common; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Globalization; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -68,6 +70,27 @@ public sealed class SqlPollReader : IDisposable private long _lastContractWarningTicks; + /// + /// Per-key last-warned instant for the per-poll warnings (R06:S-6): a frozen group's BadTimeout and a + /// failing group's query error used to log every poll, per group, indefinitely — 20 groups at 1 s was + /// 20 lines a second carrying SQL text. Keyed by the plan's SQL text, so bounded by the plan count. + /// + private readonly ConcurrentDictionary _lastWarnedTicks = new(StringComparer.Ordinal); + + /// Tags whose uncoercible cell has been warned about — once per tag per reader (the refusal rule). + private readonly ConcurrentDictionary _coerceWarned = new(StringComparer.Ordinal); + + /// + /// The resolved definitions, slot map and plans for each reference list a caller polls (R06:S-7). A + /// poll loop passes the SAME list instance every cycle, so it is planned once instead of every poll. + /// Weakly keyed: an unsubscribed list (and its entry) is collected with it. Invalidation is structural — + /// each entry remembers the definition INSTANCES it was built from, and any reference resolving to a + /// different instance (a catalog-gate swap) rebuilds it. + /// + private readonly ConditionalWeakTable, PollShape> _shapes = new(); + + private int _plansBuilt; + /// Initializes a new instance of the class. /// /// Creates the provider's connections. Passed explicitly rather than taken from @@ -182,28 +205,18 @@ public sealed class SqlPollReader : IDisposable if (results.Length == 0) return new SqlPollResult(results, 0); var readAt = DateTime.UtcNow; + var shape = GetShape(fullReferences); - // Resolve first. A miss is this tag's own problem (BadNodeIdUnknown) and must not reach the planner, - // whose members are the sole thing the slice-back walks. - var definitions = new List(fullReferences.Count); - var slots = new Dictionary>(); - for (var i = 0; i < fullReferences.Count; i++) + // A miss is this tag's own problem (BadNodeIdUnknown) and never reaches the planner, whose members + // are the sole thing the slice-back walks. + for (var i = 0; i < shape.Resolved.Length; i++) { - var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; - if (definition is null) - { + if (shape.Resolved[i] is null) results[i] = new DataValueSnapshot(null, SqlStatusCodes.BadNodeIdUnknown, null, readAt); - continue; - } - - definitions.Add(definition); - if (!slots.TryGetValue(definition, out var positions)) - slots[definition] = positions = []; - positions.Add(i); } - var rowCapExceeded = definitions.Count > 0 - ? await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false) + var rowCapExceeded = shape.Definitions.Count > 0 + ? await ReadGroupsAsync(shape, results, readAt, cancellationToken).ConfigureAwait(false) : 0; // Fail-safe. Nothing above should leave a hole, but "N in, N out" is a contract the caller indexes @@ -214,30 +227,93 @@ public sealed class SqlPollReader : IDisposable return new SqlPollResult(results, rowCapExceeded); } + /// How many times a reference list has been (re)planned. Test observation of the S-7 cache. + internal int PlansBuiltForTests => Volatile.Read(ref _plansBuilt); + /// - /// Plans the resolved definitions, runs every group concurrently, and slices the results back. - /// Returns how many groups came back over their row cap. + /// Returns the cached shape for , rebuilding it when any reference now + /// resolves to a different definition instance than the one it was built from. /// - private async Task ReadGroupsAsync( - List definitions, - Dictionary> slots, - DataValueSnapshot[] results, - DateTime readAt, - CancellationToken cancellationToken) + private PollShape GetShape(IReadOnlyList fullReferences) { - IReadOnlyList plans; + if (_shapes.TryGetValue(fullReferences, out var cached) && cached.Resolved.Length == fullReferences.Count) + { + var current = true; + for (var i = 0; i < fullReferences.Count && current; i++) + { + var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; + current = ReferenceEquals(definition, cached.Resolved[i]); + } + + if (current) return cached; + } + + var shape = BuildShape(fullReferences); + _shapes.AddOrUpdate(fullReferences, shape); + return shape; + } + + /// Resolves, slots and plans one reference list. + private PollShape BuildShape(IReadOnlyList fullReferences) + { + Interlocked.Increment(ref _plansBuilt); + + var resolved = new SqlTagDefinition?[fullReferences.Count]; + var definitions = new List(fullReferences.Count); + var slots = new Dictionary>(); + for (var i = 0; i < fullReferences.Count; i++) + { + var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; + resolved[i] = definition; + if (definition is null) continue; + + definitions.Add(definition); + if (!slots.TryGetValue(definition, out var positions)) + slots[definition] = positions = []; + positions.Add(i); + } + + if (definitions.Count == 0) return new PollShape(resolved, definitions, slots, [], PlanningFailed: false); + try { - plans = SqlGroupPlanner.Plan(definitions, _dialect); + return new PollShape(resolved, definitions, slots, SqlGroupPlanner.Plan(definitions, _dialect), PlanningFailed: false); } catch (Exception ex) when (ex is ArgumentException or NotSupportedException) { // A definition with a broken invariant — SqlEquipmentTagParser rejects all of these up front, so // reaching here means a definition was built past the parser. It is an authoring fault, not a // database fault: Bad-code the planned tags rather than throwing and triggering connection backoff. + // Logged once per (re)plan, which the cache makes once per shape rather than once per poll. _logger.LogError(ex, "Sql poll planning failed for {Count} tag(s); publishing BadConfigurationError.", definitions.Count); - foreach (var definition in definitions) + return new PollShape(resolved, definitions, slots, [], PlanningFailed: true); + } + } + + /// One reference list's resolved definitions, slot map and plans — immutable once built. + private sealed record PollShape( + SqlTagDefinition?[] Resolved, + List Definitions, + Dictionary> Slots, + IReadOnlyList Plans, + bool PlanningFailed); + + /// + /// Runs every planned group concurrently and slices the results back. Returns how many groups came + /// back over their row cap. + /// + private async Task ReadGroupsAsync( + PollShape shape, + DataValueSnapshot[] results, + DateTime readAt, + CancellationToken cancellationToken) + { + var slots = shape.Slots; + var plans = shape.Plans; + if (shape.PlanningFailed) + { + foreach (var definition in shape.Definitions) { if (!slots.TryGetValue(definition, out var positions)) continue; var snapshot = new DataValueSnapshot(null, SqlStatusCodes.BadConfigurationError, null, readAt); @@ -429,11 +505,14 @@ public sealed class SqlPollReader : IDisposable { // The table is named for the same reason the contract warnings name it: under a partial outage this // line repeats every poll, and "which source is frozen" is the only question the operator has. - _logger.LogWarning( - "Sql poll group ({Model}, '{Table}', {Members} tag(s)) exceeded the {Timeout} ms operation " + - "timeout; publishing BadTimeout.", - plan.Model, plan.Members[0].Table, plan.Members.Count, - (int)_operationTimeout.TotalMilliseconds); + if (ShouldWarn("timeout:" + plan.SqlText)) + { + _logger.LogWarning( + "Sql poll group ({Model}, '{Table}', {Members} tag(s)) exceeded the {Timeout} ms operation " + + "timeout; publishing BadTimeout. Repeats for this group are suppressed for {Interval} s.", + plan.Model, plan.Members[0].Table, plan.Members.Count, + (int)_operationTimeout.TotalMilliseconds, (int)ContractWarningInterval.TotalSeconds); + } AbandonedTask.Detach(work); return GroupResult.TimedOut; @@ -495,9 +574,13 @@ public sealed class SqlPollReader : IDisposable { // The connection opened, so the server is up — this query is what failed (an identifier that // is not in the catalog, a lock, a permission). That is this group's problem alone. - _logger.LogWarning( - ex, "Sql poll group ({Model}, {Members} tag(s)) failed: {Sql}", - plan.Model, plan.Members.Count, plan.SqlText); + if (ShouldWarn("failed:" + plan.SqlText)) + { + _logger.LogWarning( + ex, "Sql poll group ({Model}, {Members} tag(s)) failed: {Sql}. Repeats for this group are " + + "suppressed for {Interval} s.", + plan.Model, plan.Members.Count, plan.SqlText, (int)ContractWarningInterval.TotalSeconds); + } return GroupResult.Failed; } } @@ -746,9 +829,14 @@ public sealed class SqlPollReader : IDisposable var target = member.DeclaredType ?? _dialect.MapColumnType(outcome.TypeNames[valueAt]); if (!TryCoerce(cell, target, out var value)) { - _logger.LogWarning( - "Sql tag '{Tag}' could not coerce a {Actual} cell to its {Declared} type.", - member.Name, cell.GetType().Name, target); + // A per-tag refusal: warned ONCE per tag (per reader), never per poll (R06:S-6). + if (_coerceWarned.TryAdd(member.Name, 0)) + { + _logger.LogWarning( + "Sql tag '{Tag}' could not coerce a {Actual} cell to its {Declared} type; it publishes " + + "BadTypeMismatch (warned once per tag).", + member.Name, cell.GetType().Name, target); + } return new DataValueSnapshot(null, SqlStatusCodes.BadTypeMismatch, sourceTimestamp, readAt); } @@ -815,6 +903,23 @@ public sealed class SqlPollReader : IDisposable Convert.ToDateTime(cell, CultureInfo.InvariantCulture), DateTimeKind.Utc), }; + /// Rate-limits one per-poll warning by key; true at most once per . + private bool ShouldWarn(string key) + { + var now = Stopwatch.GetTimestamp(); + while (true) + { + if (!_lastWarnedTicks.TryGetValue(key, out var last)) + { + if (_lastWarnedTicks.TryAdd(key, now)) return true; + continue; + } + + if (Stopwatch.GetElapsedTime(last, now) < ContractWarningInterval) return false; + if (_lastWarnedTicks.TryUpdate(key, now, last)) return true; + } + } + /// Rate-limits the source-contract warning so a misconfigured table cannot flood the log. private bool ShouldWarnAboutContract() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index 4181449c..d26bd0b6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -605,6 +605,126 @@ public sealed class SqlPollReaderTests dialect.Factory, " ", dialect, CommandTimeout, OperationTimeout, 4, false, _ => null)); } + // ---- R06:S-7: plans are cached per subscription reference list ---- + + /// + /// A poll loop hands the reader the SAME reference list every cycle; re-planning it (and rebuilding the + /// slot dictionary keyed by a 12-field record) every poll was pure per-poll garbage. Planned once, + /// and the cached plan still reads correct values every time — the positive control. + /// + [Fact] + public async Task ReadAsync_theSameReferenceList_isPlannedOnce() + { + using var fixture = new SqlitePollFixture(); + var reader = Reader(fixture, + KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Pressure", SqlitePollFixture.SecondPresentKey)); + IReadOnlyList refs = ["Speed", "Pressure"]; + + for (var i = 0; i < 3; i++) + { + var snapshots = await reader.ReadAsync(refs, CancellationToken.None); + snapshots[0].Value.ShouldBe(SqlitePollFixture.PresentValue); + snapshots[1].Value.ShouldBe(SqlitePollFixture.SecondPresentValue); + } + + reader.PlansBuiltForTests.ShouldBe(1); + } + + /// + /// The cache is invalidated by the definitions themselves: a catalog-gate swap (or any re-resolve) + /// hands back different definition instances, and the next poll re-plans against them. + /// + [Fact] + public async Task ReadAsync_whenAReferenceResolvesToADifferentDefinition_rePlans() + { + using var fixture = new SqlitePollFixture(); + var table = new Dictionary(StringComparer.Ordinal) + { + ["Speed"] = KvTag("Speed", SqlitePollFixture.PresentKey), + }; + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: CommandTimeout, operationTimeout: OperationTimeout, + maxConcurrentGroups: 4, nullIsBad: false, resolve: r => table.GetValueOrDefault(r)); + IReadOnlyList refs = ["Speed"]; + + (await reader.ReadAsync(refs, CancellationToken.None))[0].Value.ShouldBe(SqlitePollFixture.PresentValue); + table["Speed"] = KvTag("Speed", SqlitePollFixture.SecondPresentKey); // re-pointed by a new resolve + + (await reader.ReadAsync(refs, CancellationToken.None))[0].Value.ShouldBe(SqlitePollFixture.SecondPresentValue); + reader.PlansBuiltForTests.ShouldBe(2); + } + + // ---- R06:S-6: per-poll warnings are rate-limited ---- + + /// + /// A dropped table polled every second used to log a warning — with the full SQL text — on every + /// poll, per group, forever. One per minute per plan now; every poll still publishes the Bad status. + /// The first poll's warning is the positive control. + /// + [Fact] + public async Task ReadAsync_aPersistentlyFailingGroup_warnsOncePerInterval() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = Reader(fixture, logger, + new SqlTagDefinition("Gone", SqlTagModel.KeyValue, "NoSuchTable", + KeyColumn: SqlitePollFixture.KeyColumn, KeyValue: "k", ValueColumn: SqlitePollFixture.ValueColumn)); + + for (var i = 0; i < 3; i++) + { + (await reader.ReadAsync(["Gone"], CancellationToken.None))[0] + .StatusCode.ShouldBe(SqlStatusCodes.BadCommunicationError); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning).ShouldBe(1); + } + + /// A cell that will not coerce is a per-tag refusal: one warning per tag, not one per poll. + [Fact] + public async Task ReadAsync_anUncoercibleCell_warnsOncePerTag() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = Reader(fixture, logger, + KvTag("Speed", SqlitePollFixture.PresentKey, declaredType: DriverDataType.DateTime)); + + for (var i = 0; i < 3; i++) + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0] + .StatusCode.ShouldBe(SqlStatusCodes.BadTypeMismatch); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning).ShouldBe(1); + } + + /// A frozen group's BadTimeout warning is rate-limited too; both polls still time out. + [Fact] + public async Task ReadAsync_aRepeatedlyTimedOutGroup_warnsOncePerInterval() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: TimeSpan.FromSeconds(30), operationTimeout: TimeSpan.FromMilliseconds(500), + maxConcurrentGroups: 4, nullIsBad: false, + resolve: Resolver(KvTag("Speed", SqlitePollFixture.PresentKey)), logger: logger); + + var locker = fixture.OpenNewConnection(); + Execute(locker, "BEGIN EXCLUSIVE"); + try + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + } + finally + { + Execute(locker, "ROLLBACK"); + locker.Dispose(); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("operation timeout")) + .ShouldBe(1); + } + // ---- R06:S-3: the concurrency gate is released ---- /// An idle reader releases its gate on Dispose, and a disposed reader starts no new group. From a7d21d5a659fc905c612086192a1e27f4513cc5c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:03:23 -0400 Subject: [PATCH 22/34] feat(sql): IBatchSubscribable and lifecycle-generation-stamped health (R02:P1, R06:S-9) R02:P1: SqlDriver runs PollGroupEngine, so it now implements IBatchSubscribable and raises one OnDataChangeBatch per poll burst (DriverInstanceActor forwards one message per cycle instead of one per tag). S-9: InitializeAsync/ShutdownAsync take a lifecycle generation; their health writes apply only while current, and a poll that started under an older generation (one that outlived the engine's teardown wait) no longer writes its verdict onto the new configuration. Poll-driven updates can no longer end Initializing - only the initialize's own final write does. Lifecycle writes now preserve LastSuccessfulRead. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 96 ++++++++++++---- .../SqlDriverTests.cs | 104 ++++++++++++++++++ 2 files changed, 179 insertions(+), 21 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 4a8936c9..5492c293 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -28,7 +28,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// manufacture an exception to force backoff. /// public sealed class SqlDriver - : IDriver, ITagDiscovery, IReadable, ISubscribable, IHostConnectivityProbe, ISkippedTagReporter, IAsyncDisposable + : IDriver, ITagDiscovery, IReadable, IBatchSubscribable, IHostConnectivityProbe, ISkippedTagReporter, IAsyncDisposable { /// Authored tags this driver could not build into its poll table. private readonly SkippedTagLedger _skippedTags = new(); @@ -156,6 +156,16 @@ public sealed class SqlDriver private DriverHealth _health = new(DriverState.Unknown, null, null); + /// Guards the lifecycle-generation bump against the generation-checked lifecycle health write. + private readonly object _healthLock = new(); + + /// + /// Bumped by every and . A lifecycle write from + /// a superseded call, and a poll that STARTED under an older generation (a stale poll outliving the + /// engine's 5 s teardown wait, R06:S-9), can no longer overwrite the newer state. + /// + private long _lifecycleGeneration; + /// /// The exception instance has already turned into a health verdict, held only /// until — which the poll engine calls with that same object — has @@ -173,6 +183,9 @@ public sealed class SqlDriver /// Occurs when a subscribed tag's value or quality changes. public event EventHandler? OnDataChange; + /// Occurs once per poll cycle that produced at least one change, carrying every change of that cycle. + public event EventHandler? OnDataChangeBatch; + /// Occurs when the database endpoint transitions between reachable and unreachable. public event EventHandler? OnHostStatusChanged; @@ -230,6 +243,9 @@ public sealed class SqlDriver OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)), onError: HandlePollError, backoffCap: PollBackoffCap, + // R02:P1 — one batch per poll burst, so DriverInstanceActor forwards one message per cycle instead + // of one per tag (it subscribes exactly one of the two events). + onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)), // The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this // closure IS the production consumer of its overrun signal. onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1, @@ -352,7 +368,9 @@ public sealed class SqlDriver // identifier. public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { - WriteHealth(new DriverHealth(DriverState.Initializing, null, null)); + var generation = BeginLifecycle(); + // LastSuccessfulRead is PRESERVED across the lifecycle states — it is the watchdog's evidence clock. + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); // adopt a changed config BEFORE the tag table is built, so the table and the connection it // will be polled over come from the same revision. ApplyBinding takes the options, dialect and @@ -374,7 +392,7 @@ public sealed class SqlDriver catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { // The caller tore the initialization down; not a database verdict. - WriteHealth(new DriverHealth(DriverState.Unknown, null, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, ReadHealth().LastSuccessfulRead, null)); throw; } catch (Exception ex) @@ -390,8 +408,8 @@ public sealed class SqlDriver $"Check the database is reachable and the connection string is valid."; _logger.LogError(ex, "Sql driver {DriverInstanceId} liveness check against {Endpoint} failed.", _driverInstanceId, Endpoint); - WriteHealth(new DriverHealth(DriverState.Faulted, null, message)); - TransitionHostTo(HostState.Stopped); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, message))) + TransitionHostTo(HostState.Stopped); throw new InvalidOperationException(message, ex); } @@ -404,7 +422,7 @@ public sealed class SqlDriver } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - WriteHealth(new DriverHealth(DriverState.Unknown, null, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, ReadHealth().LastSuccessfulRead, null)); throw; } catch (Exception ex) @@ -420,13 +438,13 @@ public sealed class SqlDriver _logger.LogError(ex, "Sql driver {DriverInstanceId} catalog validation against {Endpoint} failed.", _driverInstanceId, Endpoint); - WriteHealth(new DriverHealth(DriverState.Faulted, null, message)); - TransitionHostTo(HostState.Stopped); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, message))) + TransitionHostTo(HostState.Stopped); throw new InvalidOperationException(message, ex); } - WriteHealth(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); - TransitionHostTo(HostState.Running); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null))) + TransitionHostTo(HostState.Running); _logger.LogInformation( "Sql driver {DriverInstanceId} initialized against {Endpoint} with {TagCount} authored tag(s).", _driverInstanceId, Endpoint, Tags.Count); @@ -442,9 +460,10 @@ public sealed class SqlDriver /// public async Task ShutdownAsync(CancellationToken cancellationToken) { + var generation = BeginLifecycle(); var lastRead = ReadHealth().LastSuccessfulRead; await TeardownAsync().ConfigureAwait(false); - WriteHealth(new DriverHealth(DriverState.Unknown, lastRead, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, lastRead, null)); } /// @@ -517,10 +536,13 @@ public sealed class SqlDriver // unreachable" and degrade a perfectly healthy driver. ArgumentNullException.ThrowIfNull(fullReferences); + // The generation this read started under: if a Shutdown/Initialize lands while it is in flight, its + // verdict describes a superseded configuration and must not touch the newer health (R06:S-9). + var generation = Interlocked.Read(ref _lifecycleGeneration); try { var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); - ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups); + ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups, generation); return result.Snapshots; } catch (OperationCanceledException) @@ -535,8 +557,12 @@ public sealed class SqlDriver _driverInstanceId, Endpoint); // Type name only — never ex.Message (see InitializeAsync for why); the full exception is already // on the log sink via the LogWarning exception parameter above. - Degrade($"Sql driver '{_driverInstanceId}' could not reach {Endpoint}: {ex.GetType().Name}."); - TransitionHostTo(HostState.Stopped); + if (IsCurrentLifecycle(generation)) + { + Degrade($"Sql driver '{_driverInstanceId}' could not reach {Endpoint}: {ex.GetType().Name}."); + TransitionHostTo(HostState.Stopped); + } + // Mark THIS exception object as already classified before it goes back to the poll engine, which // routes it to HandlePollError next. The mark is what makes the deferral there exact — see that // method for why inferring it from the exception's type was not. @@ -614,9 +640,15 @@ public sealed class SqlDriver /// (the planner mints it before any I/O), so the count is what makes them count as the database /// answering. /// - internal void ObservePollOutcome(IReadOnlyList snapshots, int answeredRefusals = 0) + /// + /// The lifecycle generation the poll started under; a superseded one is ignored outright. Null (tests) + /// means "current". + /// + internal void ObservePollOutcome( + IReadOnlyList snapshots, int answeredRefusals = 0, long? generation = null) { if (snapshots.Count == 0) return; + if (generation is { } started && !IsCurrentLifecycle(started)) return; var good = 0; var unreachable = 0; @@ -800,7 +832,11 @@ public sealed class SqlDriver /// /// Builds the snapshot to publish from the current one. private void UpdateHealthUnlessFaulted(Func project) - => UpdateHealth(current => current.State == DriverState.Faulted ? null : project(current)); + // Initializing too: only the initialize's own final write may end it — a poll landing mid-init + // (a re-init without shutdown keeps the engine running) used to flip Initializing → Healthy. + => UpdateHealth(current => current.State is DriverState.Faulted or DriverState.Initializing + ? null + : project(current)); /// /// The one read-modify-write of the health field, as a compare-and-swap retry loop. @@ -831,13 +867,31 @@ public sealed class SqlDriver /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); + /// Starts a lifecycle operation and returns its generation; every later one supersedes it. + private long BeginLifecycle() + { + lock (_healthLock) return ++_lifecycleGeneration; + } + + /// Whether is still the newest lifecycle operation. + private bool IsCurrentLifecycle(long generation) => Interlocked.Read(ref _lifecycleGeneration) == generation; + + /// The current lifecycle generation. Test observation of the S-9 guard. + internal long LifecycleGenerationForTests => Interlocked.Read(ref _lifecycleGeneration); + /// - /// Unconditional publish of a new health snapshot, for the lifecycle transitions that are - /// last-writer-wins by definition (Initialize / Shutdown). Everything driven by a poll goes through - /// instead, because it has to read the current snapshot first. + /// A lifecycle write that applies only while is current — atomic against + /// , so a superseded initialize/shutdown never overwrites a newer one. /// - /// The health snapshot to publish. - private void WriteHealth(DriverHealth value) => Volatile.Write(ref _health, value); + private bool WriteHealthIfCurrent(long generation, DriverHealth value) + { + lock (_healthLock) + { + if (_lifecycleGeneration != generation) return false; + Volatile.Write(ref _health, value); + return true; + } + } /// Records a host-state change and raises — on transitions only. private void TransitionHostTo(HostState newState) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 6dd21ce9..320a0a0f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -385,6 +385,85 @@ public sealed class SqlDriverTests driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); } + /// + /// R06:S-9 / lifecycle generations. A poll that started before a Shutdown → Initialize cycle + /// (it can outlive the engine's 5 s teardown wait) must not write its verdict onto the NEW + /// configuration's health. Control: the same BadTimeout verdict under the current generation degrades. + /// + [Fact] + public async Task ObservePollOutcome_fromASupersededGeneration_isIgnored() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + var stale = driver.LifecycleGenerationForTests; + + await driver.ShutdownAsync(CancellationToken.None); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + var timedOut = new DataValueSnapshot(null, SqlStatusCodes.BadTimeout, null, DateTime.UtcNow); + + driver.ObservePollOutcome([timedOut], generation: stale); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a superseded poll must not degrade the new init"); + + driver.ObservePollOutcome([timedOut], generation: driver.LifecycleGenerationForTests); + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + /// + /// Only the initialize's own final write ends Initializing: a Good poll landing mid-init used to + /// flip it to Healthy before liveness had even answered. Control: once the init completes it is Healthy. + /// + [Fact] + public async Task APollLandingMidInitialize_cannotEndInitializing() + { + using var fixture = new SqlitePollFixture(); + var factory = new GatedFactory(); + await using var driver = NewDriver(fixture, fixture.ConnectionString, CapturingLogger.Null, factory, + KvEntry("Speed", SqlitePollFixture.PresentKey)); + + factory.Arm(); + var init = driver.InitializeAsync(ConfigJson, CancellationToken.None); + (await factory.Entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBeTrue(); + driver.GetHealth().State.ShouldBe(DriverState.Initializing); + + driver.ObservePollOutcome([new DataValueSnapshot(42.0, SqlStatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow)]); + driver.GetHealth().State.ShouldBe(DriverState.Initializing); + + factory.Release(); + await init.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// A SQLite factory whose next connection creation blocks until released — parks an initialize. + private sealed class GatedFactory : DbProviderFactory + { + private readonly ManualResetEventSlim _release = new(initialState: true); + private volatile bool _armed; + + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Arm() + { + _release.Reset(); + _armed = true; + } + + public void Release() => _release.Set(); + + public override DbConnection CreateConnection() + { + if (_armed) + { + _armed = false; + Entered.TrySetResult(true); + _release.Wait(TimeSpan.FromSeconds(30)); + } + + return Microsoft.Data.Sqlite.SqliteFactory.Instance.CreateConnection()!; + } + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] @@ -474,6 +553,31 @@ public sealed class SqlDriverTests Volatile.Read(ref changes).ShouldBe(afterUnsubscribe); } + /// + /// R02:P1. The driver runs PollGroupEngine, so it raises one batch per poll burst: + /// DriverInstanceActor then forwards one message per cycle instead of one per tag. Both tags of + /// the first (force-raised) poll arrive in the SAME batch, on the subscription's handle. + /// + [Fact] + public async Task Subscribe_raisesOneBatchPerPollBurst() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, + KvEntry("Speed", SqlitePollFixture.PresentKey), KvEntry("Pressure", SqlitePollFixture.SecondPresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + + var batchable = driver.ShouldBeAssignableTo(); + var first = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + batchable!.OnDataChangeBatch += (_, args) => first.TrySetResult(args); + + var handle = await driver.SubscribeAsync(["Speed", "Pressure"], TimeSpan.FromMilliseconds(100), CancellationToken.None); + + var batch = await first.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + batch.SubscriptionHandle.ShouldBe(handle); + batch.Items.Select(i => i.FullReference).OrderBy(r => r, StringComparer.Ordinal).ShouldBe(["Pressure", "Speed"]); + batch.Items.Single(i => i.FullReference == "Speed").Snapshot.Value.ShouldBe(SqlitePollFixture.PresentValue); + } + // ---- disposal ---- [Fact] From 2a28c06fc790fe5d985e27e427a7879af3cab03c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:04:59 -0400 Subject: [PATCH 23/34] fix(sql): a skipped tag reports the deploy gate's reason, pinned for the deferred Query model (R06:S-4) A model:Query tag reached the runtime skip ledger as a generic 'did not map'. The ledger reason (and the skip warning) now comes from SqlTagDefinitionFactory.Inspect - the deploy gate's own explanation over the same parser - so /hosts names the deferral. A test pins that the deploy gate names Query through the parser. Making it a hard deploy error in Warn mode is a ControlPlane change (follow-up); MySql/Odbc and allowWrites remain feature issues. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 11 +++++++--- .../SqlDriverTests.cs | 20 +++++++++++++++++++ .../SqlEquipmentTagParserTests.cs | 15 ++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 5492c293..7fbd99d7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -929,10 +929,15 @@ public sealed class SqlDriver table[entry.RawPath] = definition; else { - blobSkips.Add(new SkippedTag(entry.RawPath, "TagConfig did not map to a definition")); + // The reason is the DEPLOY GATE's own explanation (R06:S-4) — SqlTagDefinitionFactory.Inspect + // asks this same parser and then says why — so /hosts names e.g. the deferred Query model + // instead of a generic "did not map" for a tag the gate already warned about. + var reason = SqlTagDefinitionFactory.Inspect(entry.TagConfig).FirstOrDefault() + ?? "TagConfig did not map to a definition"; + blobSkips.Add(new SkippedTag(entry.RawPath, reason)); _logger.LogWarning( - "Sql tag config did not map to a definition; skipping. Driver={DriverInstanceId} RawPath={RawPath}", - _driverInstanceId, entry.RawPath); + "Sql tag config did not map to a definition; skipping. Driver={DriverInstanceId} RawPath={RawPath} Reason={Reason}", + _driverInstanceId, entry.RawPath, reason); } } catch (Exception ex) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 320a0a0f..1eec94e5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -464,6 +464,26 @@ public sealed class SqlDriverTests } } + /// + /// R06:S-4. A skipped tag's ledger reason (what /hosts shows) is the deploy gate's own explanation + /// — for a Query tag, that the model is deferred — instead of a generic "did not map". Control: a + /// well-formed tag is not skipped at all. + /// + [Fact] + public async Task ADeferredQueryTag_isSkipped_withTheGatesReason() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, + KvEntry("Speed", SqlitePollFixture.PresentKey), + new RawTagEntry("Q", """{"driver":"Sql","model":"Query","table":"T","query":"SELECT 1"}""", WriteIdempotent: false)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + + var report = ((ISkippedTagReporter)driver).GetSkippedTags().ShouldNotBeNull(); + var skipped = report.Sample.ShouldHaveSingleItem(); + skipped.RawPath.ShouldBe("Q"); + skipped.Reason.ShouldContain("Query"); + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs index 4734ee30..3433a4af 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs @@ -232,6 +232,21 @@ public class SqlEquipmentTagParserTests SqlEquipmentTagParser.TryParse(json, "raw", out _).ShouldBeFalse(); } + /// + /// R06:S-4. The deploy gate (EquipmentTagConfigInspector) reaches this through the parser: + /// a Query tag draws a warning naming the deferral (a rejection under TagConfigValidation=Error). + /// Control: a served key-value blob draws nothing. + /// + [Fact] + public void Inspect_theDeferredQueryModel_isNamedAtTheDeployGate() + { + SqlTagDefinitionFactory.Inspect("""{"driver":"Sql","model":"Query","table":"T","query":"SELECT 1"}""") + .ShouldHaveSingleItem().ShouldContain("Query"); + SqlTagDefinitionFactory.Inspect( + """{"driver":"Sql","model":"KeyValue","table":"T","keyColumn":"k","keyValue":"v","valueColumn":"c"}""") + .ShouldBeEmpty(); + } + [Theory] [InlineData("""{"driver":"Sql","model":"KeyValue","keyColumn":"k","keyValue":"v","valueColumn":"c"}""")] [InlineData("""{"driver":"Sql","model":"KeyValue","table":"","keyColumn":"k","keyValue":"v","valueColumn":"c"}""")] From 3531834ccedc756d21b47df6369494ad1498ccf5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:06:02 -0400 Subject: [PATCH 24/34] fix(modbus): review r1 - resolved address keys win only when the editor pinned them N1: the m2 precedence trusted region/address whenever the keys were present, but the AdminUI model stamps defaulted HoldingRegisters/0 on every save, so a string-only tag (CSV import, or any editor toggle) silently became HR[0] - reads AND writes. The resolved keys are now authoritative only when the editor marked them ("addressResolved": true, written by Resolve); an unmarked addressString is always resolved through the parser with the driver's family. The model no longer stamps address fields for a string-only tag, CSV export leaves its address cells blank, and the editor hint distinguishes string-only from pinned tags. Tests: CSV import and editor save of a string-only tag, a stamped-default blob, and the pinned positive control across families. n1: docs/v2/modbus-addressing.md updated (FC23 opt-in, maxReadGap semantics, coalescing scope, addressString/family rules, coil coercion, refusals). n2: form round-trip tests for UseFC23ForMultiRegisterWrites / DisableReadCoalescing. n4: the editor keeps unresolved address text when the parent echoes back its own emit. n3: noted, no change. Refs #741 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/v2/modbus-addressing.md | 35 +++++++++++-- .../ModbusTagDefinitionFactory.cs | 35 ++++++++++--- .../TagEditors/ModbusTagConfigEditor.razor | 30 ++++++++--- .../Uns/TagEditors/CsvColumnMap.cs | 15 +++--- .../Uns/TagEditors/ModbusTagConfigModel.cs | 48 +++++++++++++---- .../ModbusFamilyWiringTests.cs | 19 ++++++- .../ModbusDriverPageFormSerializationTests.cs | 4 ++ .../Uns/ModbusDriverFormModelTests.cs | 17 ++++++ .../Uns/ModbusTagConfigModelTests.cs | 52 +++++++++++++++++++ 9 files changed, 219 insertions(+), 36 deletions(-) diff --git a/docs/v2/modbus-addressing.md b/docs/v2/modbus-addressing.md index 20950a38..42f3fab8 100644 --- a/docs/v2/modbus-addressing.md +++ b/docs/v2/modbus-addressing.md @@ -149,7 +149,11 @@ Beyond per-tag addressing, `ModbusDriverOptions` exposes (#139–#143): - `UseFC15ForSingleCoilWrites` — force FC15 (write multiple coils qty=1) for single-coil writes. Safety/audit PLCs may require this. - `UseFC16ForSingleRegisterWrites` — same for FC16 vs FC06. -- `DisableFC23` — kill switch for FC23 (currently unused; reserved). +- `UseFC23ForMultiRegisterWrites` (default `false`) — opt-in: a 2–121-register + holding write is sent as one FC23 (write + read-back). A unit answering + exception 01 moves to FC16 until the next re-init. Without the key the + wire stays FC16 (#741). +- `DisableFC23` — hard kill switch: never send FC23, even when opted in. ### Subscribe (#141) - Per-tag `Deadband` — suppress sub-threshold publishes on numeric tags. @@ -165,9 +169,13 @@ Beyond per-tag addressing, `ModbusDriverOptions` exposes (#139–#143): this tag in isolation regardless of `MaxReadGap`). ### Block-read coalescing (#143) -- `MaxReadGap` (default 0 = off) — gap budget the planner is willing to - bridge between adjacent register tags. With `MaxReadGap=10`, three tags - at HR 100/102/110 collapse into one FC03 of quantity 11. +- `MaxReadGap` — gap budget the planner is willing to bridge between + adjacent tags of one (unit, region). With `MaxReadGap=10`, three tags at + HR 100/102/110 collapse into one FC03 of quantity 11. An explicit `0` = + coalescing off (its historical meaning); the default `8` applies only when + the key is absent (#741). Coils / DiscreteInputs (gap in bits), `String` + scalars and same-word `BitInRegister` tags coalesce too. +- `DisableReadCoalescing` — turn coalescing off regardless of the gap. ### Coalescing auto-recovery (#148 / #150 / #151 / #152) - A coalesced read that fails with a Modbus exception (write-only or @@ -231,3 +239,22 @@ the current Kepware "Modbus Ethernet Driver Help" PDF and Ignition's "Modbus Addressing" user-manual page — if a critical tool's mnemonics have shifted, add aliases in `ModbusAddressParser.TryParseType` rather than asking users to rewrite spreadsheets. + + +## Address strings, families and region/type rules (#741) + +- A tag's `addressString` is resolved through `ModbusAddressParser` with the + **driver's** `Family` / `MelsecSubFamily`. When the AdminUI editor resolves + it ("Resolve with driver family") it persists the resolved + `region`/`address`/type fields plus `"addressResolved": true`; those pinned + fields are then authoritative, so a later family change never remaps the + tag. A blob with an **unmarked** `addressString` (CSV / API authoring) is + always resolved from the string — any `region`/`address` beside it is + ignored, so it can never collapse onto HR[0]. Editing a resolved field by + hand drops the string. +- Coils / DiscreteInputs carry only Bool. A bit-region tag persisted with a + register `dataType` (the pre-#741 editor saved `Int16`) is coerced to Bool, + never refused. A register region with `Bool` is refused (use + `BitInRegister`). +- Exception 01/02/03 are device refusals: per-tag Bad, one Warning per tag, + no driver Degraded; reads and writes share one comms-fault health slot. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs index 633d6931..32b4adf6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Contracts/ModbusTagDefinitionFactory.cs @@ -35,10 +35,11 @@ public static class ModbusTagDefinitionFactory => FromTagConfig(tagConfig, rawPath, ModbusFamily.Generic, MelsecFamily.Q_L_iQR, out def); /// - /// Family-aware overload. The persisted region + address keys are AUTHORITATIVE whenever - /// present — the editor resolves an addressString with the driver's family and persists the - /// result, so a later Family / MelsecSubFamily change can never silently remap a live tag. A blob that - /// carries ONLY an addressString (CSV / API authoring, the Modicon / mnemonic / family-native + /// Family-aware overload. The persisted region + address keys are AUTHORITATIVE when they + /// were actually authored — a classic tag with no addressString, or one the editor resolved and + /// marked () — so a later Family / MelsecSubFamily change can never + /// silently remap such a tag. A blob whose addressString is unmarked (CSV / API authoring, even if + /// a model round-trip stamped default region/address beside it; the Modicon / mnemonic / family-native /// grammar of — 40001:F:CDAB, DL205 V2000, MELSEC /// D100) is resolved here with the DRIVER's / /// . Without an addressString this is exactly the 3-argument @@ -69,7 +70,7 @@ public static class ModbusTagDefinitionFactory ushort stringLength; int? arrayCount; var addressText = ReadAddressString(root, out var addressStringInvalid); - var hasResolvedKeys = root.TryGetProperty("region", out _) && root.TryGetProperty("address", out _); + var hasResolvedKeys = ResolvedKeysAuthoritative(root, addressText); if (!hasResolvedKeys && addressStringInvalid) return false; if (!hasResolvedKeys && addressText is not null) { @@ -174,6 +175,28 @@ public static class ModbusTagDefinitionFactory (ModbusFamily.MELSEC, MelsecFamily.F_iQF), ]; + /// + /// The key the tag editor writes beside the resolved region / address when it resolved an + /// addressString with the driver's family ("addressResolved": true). Its presence — not the + /// mere presence of region / address — is what makes the resolved keys authoritative over + /// the string: the AdminUI model stamps DEFAULTED region/address (HoldingRegisters / 0) on + /// every save, so key presence alone turned a string-only CSV/API tag into HR[0] on its first edit. + /// + public const string AddressResolvedKey = "addressResolved"; + + /// + /// Whether the persisted region / address are the tag's real address: they are present AND + /// either there is no addressString (a classic region + address tag) or the editor marked them as + /// resolved from it (). A blob carrying an addressString without + /// the marker resolves the string through the parser, never the (possibly defaulted) keys. + /// + private static bool ResolvedKeysAuthoritative(JsonElement root, string? addressText) + { + if (!root.TryGetProperty("region", out _) || !root.TryGetProperty("address", out _)) return false; + if (addressText is null) return true; + return root.TryGetProperty(AddressResolvedKey, out var marker) && marker.ValueKind == JsonValueKind.True; + } + /// Reads the optional addressString: null when absent or blank; flags a non-string value. private static string? ReadAddressString(JsonElement root, out bool invalid) { @@ -320,7 +343,7 @@ public static class ModbusTagDefinitionFactory private static void DescribeRejection(JsonElement root, List warnings) { var addressText = ReadAddressString(root, out var addressStringInvalid); - var hasResolvedKeys = root.TryGetProperty("region", out _) && root.TryGetProperty("address", out _); + var hasResolvedKeys = ResolvedKeysAuthoritative(root, addressText); if (!hasResolvedKeys && addressStringInvalid) { warnings.Add("Modbus TagConfig 'addressString' is not a string — the tag will not resolve (BadNodeIdUnknown)."); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor index 33eae3fe..42fb01eb 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Components/Shared/Uns/TagEditors/ModbusTagConfigEditor.razor @@ -14,9 +14,13 @@ {
@_addressError
} + else if (_m.IsStringOnly) + { +
String-only tag: the driver resolves @_m.AddressString with its own family (here @_family). The fields below are NOT its address — press Resolve to pin them, or edit a field to replace the string.
+ } else if (_m.AddressString is not null) { -
Resolved from @_m.AddressString (family @_family) — the driver re-resolves it with its own family. Editing a field below drops it.
+
Resolved from @_m.AddressString (family @_family) and pinned: the fields below are what the driver polls, so a later family change does not remap the tag. Editing a field drops the string.
}
@@ -96,7 +110,7 @@ _addressText = _m.AddressString; _addressError = null; } - _family = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()).Family; + (_family, _melsecSub) = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()); } private static int ParseInt(object? v, int fallback = 0) => int.TryParse(v?.ToString(), out var i) ? i : fallback; @@ -122,6 +136,7 @@ private string? _addressText; private string? _addressError; private ModbusFamily _family = ModbusFamily.Generic; + private MelsecFamily _melsecSub = MelsecFamily.Q_L_iQR; // A hand edit of any resolved field makes those fields authoritative: drop the address string, which the // driver would otherwise re-resolve over them. @@ -132,7 +147,7 @@ private async Task ResolveAddressAsync() { var (family, sub) = ModbusTagConfigModel.ReadFamily(GetDriverConfigJson()); - _family = family; + (_family, _melsecSub) = (family, sub); var error = _m.ResolveAddress(_addressText, family, sub); _addressError = error; if (error is null) await EmitAsync(); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/RawTagCsvMapper.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/RawTagCsvMapper.cs index 4460619d..b521cf58 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/RawTagCsvMapper.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/RawTagCsvMapper.cs @@ -188,6 +188,24 @@ public static class RawTagCsvMapper var json = map?.ApplyTypedColumns(baseJson.Length == 0 ? "{}" : baseJson, row) ?? (baseJson.Length == 0 ? "{}" : baseJson); + // A typed column that handed base keys to the model (CsvTypedColumn.ClearsKeys — Modbus's address + // cells and the addressString they contradict) and thereby DROPPED one is reported like any other + // override, so the operator sees the string go rather than losing it silently (review r3 m1). + if (map is not null) + { + var mergedKeys = ReadRootKeys(json, out _); + foreach (var col in map.Columns) + { + if (typedUsed.Contains(col.Header) + && col.ClearsKeys is { Count: > 0 } handed + && handed.Any(k => baseKeys.Contains(k) && !mergedKeys.Contains(k)) + && !overrodeFallback.Contains(col.Header)) + { + overrodeFallback.Add(col.Header); + } + } + } + // Historize flags. var isHistorized = ParseFlag("IsHistorized", Cell("IsHistorized")); json = TagHistorizeConfig.Set(json, isHistorized, Cell("HistorianTagname")); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/CsvColumnMap.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/CsvColumnMap.cs index c923f0f8..e58c8f7f 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/CsvColumnMap.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/CsvColumnMap.cs @@ -29,10 +29,11 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Uns.TagEditors; /// path has to know the whole set and not just the one it writes. /// /// -/// Root keys a NON-BLANK cell in this column invalidates: they are neither restored from the base cell nor -/// owned for export. Modbus's address columns use it for addressString / addressResolved — a -/// typed address cell is an authored address, exactly like a hand edit in the editor, so a stale address -/// string (and its pinned marker) must not survive beside it. +/// Root keys a NON-BLANK cell in this column hands to the MODEL: they are not restored from the base cell +/// (the model's serialisation decides them), and they are not owned for export. Modbus's address columns use +/// it for addressString / addressResolved — a typed address cell is an authored key address, and +/// the model keeps the string (and marker) only while it still describes the cells, dropping one a cell +/// contradicts. The import review grid reports a column that dropped such a key. /// public sealed record CsvTypedColumn( string Header, @@ -168,22 +169,28 @@ public sealed class ModelBackedCsvDriverTagMap : CsvDriverTagMap private readonly Func _fromJson; private readonly Func _toJson; private readonly IReadOnlyList> _columns; + private readonly Action>? _afterTypedColumns; /// Builds a model-backed map. /// The driver-type string served. /// The model's FromJson loader (preserves unknown keys). /// The model's ToJson serialiser. /// The typed columns (metadata + read/write pairs). + /// Optional model hook run once after a row's non-blank typed cells were applied + /// (given the columns that fired), before serialisation. Modbus uses it to reconcile the address string with + /// the authored address cells. public ModelBackedCsvDriverTagMap( string driverType, Func fromJson, Func toJson, - IReadOnlyList> columns) + IReadOnlyList> columns, + Action>? afterTypedColumns = null) { DriverType = driverType; _fromJson = fromJson; _toJson = toJson; _columns = columns; + _afterTypedColumns = afterTypedColumns; Columns = columns.Select(c => c.Meta).ToArray(); } @@ -234,6 +241,8 @@ public sealed class ModelBackedCsvDriverTagMap : CsvDriverTagMap } } + if (fired.Count > 0) { _afterTypedColumns?.Invoke(model, fired); } + var merged = TagConfigJson.ParseOrNew(_toJson(model)); foreach (var (key, node) in baseObj) { @@ -467,30 +476,35 @@ public static class CsvColumnMap { var maps = new CsvDriverTagMap[] { - // A string-only Modbus tag (addressString, no addressResolved marker) exports BLANK address cells: - // its model fields are load-time defaults, and exporting them as HoldingRegisters / 0 invited a - // re-import that looked like an authored HR[0] address. The driver resolves the string. + // A Modbus tag whose blob has no region + address keys (string-only, or naming no usable address — the + // one authority, ModbusTagAddressResolution, decides) exports BLANK address cells: its model fields are + // load-time defaults, and exporting them as HoldingRegisters / 0 invited a re-import that authored an + // HR[0] address. The driver resolves the string (or refuses the tag). new ModelBackedCsvDriverTagMap( DriverTypeNames.Modbus, ModbusTagConfigModel.FromJson, m => m.ToJson(), new[] { - // The six address columns: a non-blank cell is an AUTHORED address (like a hand edit in the - // editor), so it drops the address string and its pinned marker (ClearAddressString + - // ClearsKeys) — otherwise a string-only row silently discarded the cell. + // The six address columns: a non-blank cell is an AUTHORED key address. The string and its + // marker survive only while they still describe the cells (ReconcileAddressStringWithKeys, + // below) — so an unchanged export → import is lossless and a contradicting cell drops them. ModbusAddrCol("Region", "region", nameof(ModbusTagConfigModel.Region), - m => m.IsStringOnly ? null : m.Region.ToString(), (m, s) => m.Region = CsvValues.ParseEnum("Region", s)), + m => !m.HasKeyAddress ? null : m.Region.ToString(), (m, s) => m.Region = CsvValues.ParseEnum("Region", s)), ModbusAddrCol("Address", "address", nameof(ModbusTagConfigModel.Address), - m => m.IsStringOnly ? null : m.Address.ToString(CultureInfo.InvariantCulture), (m, s) => m.Address = CsvValues.ParseInt("Address", s)), + m => !m.HasKeyAddress ? null : m.Address.ToString(CultureInfo.InvariantCulture), (m, s) => m.Address = CsvValues.ParseInt("Address", s)), ModbusAddrCol("ModbusDataType", "dataType", nameof(ModbusTagConfigModel.DataType), - m => m.IsStringOnly ? null : m.DataType.ToString(), (m, s) => m.DataType = CsvValues.ParseEnum("ModbusDataType", s)), + m => !m.HasKeyAddress ? null : m.DataType.ToString(), (m, s) => m.DataType = CsvValues.ParseEnum("ModbusDataType", s)), ModbusAddrCol("ByteOrder", "byteOrder", nameof(ModbusTagConfigModel.ByteOrder), - m => m.IsStringOnly ? null : m.ByteOrder.ToString(), (m, s) => m.ByteOrder = CsvValues.ParseEnum("ByteOrder", s)), + m => !m.HasKeyAddress ? null : m.ByteOrder.ToString(), (m, s) => m.ByteOrder = CsvValues.ParseEnum("ByteOrder", s)), ModbusAddrCol("BitIndex", "bitIndex", nameof(ModbusTagConfigModel.BitIndex), - m => m.IsStringOnly ? null : m.BitIndex.ToString(CultureInfo.InvariantCulture), (m, s) => m.BitIndex = CsvValues.ParseInt("BitIndex", s)), + m => !m.HasKeyAddress ? null : m.BitIndex.ToString(CultureInfo.InvariantCulture), (m, s) => m.BitIndex = CsvValues.ParseInt("BitIndex", s)), ModbusAddrCol("StringLength", "stringLength", nameof(ModbusTagConfigModel.StringLength), - m => m.IsStringOnly ? null : m.StringLength.ToString(CultureInfo.InvariantCulture), (m, s) => m.StringLength = CsvValues.ParseInt("StringLength", s)), + m => !m.HasKeyAddress ? null : m.StringLength.ToString(CultureInfo.InvariantCulture), (m, s) => m.StringLength = CsvValues.ParseInt("StringLength", s)), ModbusCol("Writable", "writable", nameof(ModbusTagConfigModel.Writable), m => CsvValues.FormatNullableBool(m.Writable), (m, s) => m.Writable = CsvValues.ParseBool("Writable", s)), + }, + (m, fired) => + { + if (fired.Any(c => c.ClearsKeys is { Count: > 0 })) { m.ReconcileAddressStringWithKeys(); } }), new ModelBackedCsvDriverTagMap( @@ -634,7 +648,7 @@ public static class CsvColumnMap private static CsvModelColumn ModbusAddrCol( string h, string k, string p, Func get, Action set) => new(new CsvTypedColumn(h, k, p, ClearsKeys: ["addressString", ModbusTagDefinitionFactory.AddressResolvedKey]), - get, (m, s) => { m.ClearAddressString(); set(m, s); }); + get, set); private static CsvModelColumn S7Col( string h, string k, string p, Func get, Action set) diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs index 516eb3c6..b621db2b 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.AdminUI/Uns/TagEditors/ModbusTagConfigModel.cs @@ -7,23 +7,32 @@ namespace ZB.MOM.WW.OtOpcUa.AdminUI.Uns.TagEditors; /// fields; name/writable live on the Tag entity). Preserves unrecognised JSON keys across a load→save.
 public sealed class ModbusTagConfigModel { + // Every address field is backed so that SETTING one is an authored address (see _authorsKeys): a hand edit, + // a CSV address cell, ResolveAddress or an object initialiser. FromJson assigns the backing fields directly. + private ModbusRegion _region = ModbusRegion.HoldingRegisters; + private int _address; + private ModbusDataType _dataType = ModbusDataType.Int16; + private ModbusByteOrder _byteOrder = ModbusByteOrder.BigEndian; + private int _bitIndex; + private int _stringLength; + /// Register region (HoldingRegisters/InputRegisters/Coils/DiscreteInputs). - public ModbusRegion Region { get; set; } = ModbusRegion.HoldingRegisters; + public ModbusRegion Region { get => _region; set { _region = value; _authorsKeys = true; } } /// Starting register/coil address. - public int Address { get; set; } + public int Address { get => _address; set { _address = value; _authorsKeys = true; } } /// Wire value type the register block decodes to. - public ModbusDataType DataType { get; set; } = ModbusDataType.Int16; + public ModbusDataType DataType { get => _dataType; set { _dataType = value; _authorsKeys = true; } } /// Word/byte ordering for multi-register values. - public ModbusByteOrder ByteOrder { get; set; } = ModbusByteOrder.BigEndian; + public ModbusByteOrder ByteOrder { get => _byteOrder; set { _byteOrder = value; _authorsKeys = true; } } /// Bit index (0-15) for BitInRegister tags. - public int BitIndex { get; set; } + public int BitIndex { get => _bitIndex; set { _bitIndex = value; _authorsKeys = true; } } /// String length in characters for String tags. - public int StringLength { get; set; } + public int StringLength { get => _stringLength; set { _stringLength = value; _authorsKeys = true; } } /// Optional writable flag. null ⇒ the key is omitted (driver default true); /// an explicit false makes the equipment tag read-only. Preserved across load→save so an absent @@ -31,37 +40,61 @@ public sealed class ModbusTagConfigModel public bool? Writable { get; set; } /// - /// The address in the parser grammar (40001:F:CDAB, DL205 V2000, MELSEC D100) when - /// the tag was authored that way; null for a tag authored by region + numeric address. When - /// the persisted resolved fields are what the driver reads; otherwise the - /// driver resolves the string with its own family (see ). Editing any resolved - /// field by hand clears it () so the fields become the authored address. + /// The address in the parser grammar (40001:F:CDAB, DL205 V2000, MELSEC D100) when the + /// tag carries one; null otherwise. Whether it IS the address is not this model's call — it is + /// , asked of : the string is the address + /// only when the saved blob has no region + address keys (). /// public string? AddressString { get; set; } /// - /// True when the region / offset / type fields were resolved FROM by - /// (persisted as "addressResolved": true). Only then are the persisted - /// fields authoritative over the string; an unmarked string is resolved by the driver with its own family. + /// The editor's "addressResolved": true marker: the keys were resolved FROM + /// by . Display / CSV state only — it never makes keys authoritative (keys win + /// by presence) and a marker on a blob without keys is ignored (review r3 N3). /// public bool AddressResolved { get; set; } /// - /// A string-only tag (CSV / API authored): it carries an the driver resolves, - /// and its region / offset / type fields are NOT the tag's address — never stamps - /// them, so a round-trip (CSV import, a Writable toggle) cannot turn the tag into HoldingRegisters[0]. + /// Which part of the blob would save names the address — the one authority's verdict + /// () on this model's own output, so the editor, + /// the CSV export and the driver can never disagree about it. /// - public bool IsStringOnly => AddressString is not null && !AddressResolved && !_loadedWithAddressKeys; + public ModbusAddressShape AddressShape => ModbusTagAddressResolution.Classify(ToJson()); /// - /// A row that carries an BESIDE region/address keys without the pinned marker - /// — a pre-#741 shape (the old driver never read the string). Its keys are the address (the driver loads - /// from them, as it always did); the string is informational only. + /// A string-only tag (CSV / API authored, no region + address keys): the driver resolves + /// with its own family, and the region / offset / type fields are NOT the address + /// — never stamps them, so no round trip can turn the tag into HoldingRegisters[0]. /// - public bool HasLegacyAddressString => AddressString is not null && !AddressResolved && _loadedWithAddressKeys; + public bool IsStringOnly => AddressShape == ModbusAddressShape.StringOnly; - // Whether the blob this model was loaded from carried BOTH region and address keys. - private bool _loadedWithAddressKeys; + /// + /// Whether the fields are the tag's address: the saved blob carries region + address. False for + /// a string-only tag and for a blob that names no usable address (the fields are then load-time defaults + /// and must not be presented — or exported — as an address). + /// + public bool HasKeyAddress => AddressShape == ModbusAddressShape.Keys; + + /// The string was resolved into the keys and pinned (marker + keys). + public bool IsPinned => AddressString is not null && AddressResolved && HasKeyAddress; + + /// + /// Keys beside an unmarked — a pre-#741 row (the old driver never read the + /// string). The keys are the address; the string is informational only. + /// + public bool HasLegacyAddressString => AddressString is not null && !AddressResolved && HasKeyAddress; + + /// What the driver serves this tag as under a family — the one authority, on the blob this model saves. + /// The driver's PLC family. + /// The driver's MELSEC sub-family. + /// The resolution (refused when the family cannot parse a string-only tag). + public ModbusAddressResolution ResolveWith(ModbusFamily family, MelsecFamily melsecSubFamily) + => ModbusTagAddressResolution.Resolve(ToJson(), family, melsecSubFamily); + + // Whether ToJson writes the six address fields: the loaded blob carried the keys (or named no address at all + // AND the operator then authored a field), or an address field was set since. Never true for a string-only or + // invalid-string blob that nobody edited: the editor must not write default keys it did not resolve. + private bool _authorsKeys = true; /// /// Resolves an address string through with the owning driver's @@ -79,6 +112,7 @@ public sealed class ModbusTagConfigModel return error ?? "Address did not parse."; AddressString = text!.Trim(); AddressResolved = true; + _authorsKeys = true; Region = parsed.Region; Address = parsed.Offset; DataType = parsed.DataType; @@ -103,6 +137,20 @@ public sealed class ModbusTagConfigModel { AddressString = null; AddressResolved = false; + _authorsKeys = true; + } + + /// + /// After a CSV import's typed address cells were applied (they are an authored key address): keeps the + /// (and its marker) only when it still describes those fields under some family + /// — so an unchanged export → import of a pinned row is lossless (review r3 m1) — and drops it when a cell + /// contradicts it. Either way the keys are the address; this only decides the metadata. + /// + public void ReconcileAddressStringWithKeys() + { + if (AddressString is null) return; + if (!ModbusTagAddressResolution.AddressStringDescribes(AddressString, Region, Address, DataType, ByteOrder, BitIndex, StringLength)) + ClearAddressString(); } /// @@ -155,25 +203,29 @@ public sealed class ModbusTagConfigModel public static ModbusTagConfigModel FromJson(string? json) { var o = TagConfigJson.ParseOrNew(json); + var shape = ModbusTagAddressResolution.Classify(TagConfigJson.Serialize(o)); + var region = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters); return new ModbusTagConfigModel { _source = json ?? "", - Region = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters), - Address = TagConfigJson.GetInt(o, "address"), + _region = region, + _address = TagConfigJson.GetInt(o, "address"), // Absent dataType defaults per region, exactly as the driver's factory does (Bool for a bit region), // and a bit region shows Bool whatever was persisted — the driver coerces a legacy coil/DI // "dataType":"Int16" (what the pre-#741 editor saved) to Bool, so the editor shows what is polled. - DataType = TagConfigJson.GetEnum(o, "region", ModbusRegion.HoldingRegisters) is ModbusRegion.Coils or ModbusRegion.DiscreteInputs + _dataType = region is ModbusRegion.Coils or ModbusRegion.DiscreteInputs ? ModbusDataType.Bool : TagConfigJson.GetEnum(o, "dataType", ModbusDataType.Int16), - ByteOrder = TagConfigJson.GetEnum(o, "byteOrder", ModbusByteOrder.BigEndian), - BitIndex = TagConfigJson.GetInt(o, "bitIndex"), - StringLength = TagConfigJson.GetInt(o, "stringLength"), + _byteOrder = TagConfigJson.GetEnum(o, "byteOrder", ModbusByteOrder.BigEndian), + _bitIndex = TagConfigJson.GetInt(o, "bitIndex"), + _stringLength = TagConfigJson.GetInt(o, "stringLength"), Writable = TagConfigJson.GetBoolNullable(o, "writable"), - AddressString = string.IsNullOrWhiteSpace(TagConfigJson.GetString(o, "addressString")) - ? null : TagConfigJson.GetString(o, "addressString"), - AddressResolved = TagConfigJson.GetBoolNullable(o, ModbusTagDefinitionFactory.AddressResolvedKey) == true, - _loadedWithAddressKeys = o.ContainsKey("region") && o.ContainsKey("address"), + AddressString = string.IsNullOrWhiteSpace(TagConfigJson.GetString(o, ModbusTagAddressResolution.AddressStringKey)) + ? null : TagConfigJson.GetString(o, ModbusTagAddressResolution.AddressStringKey), + AddressResolved = TagConfigJson.GetBoolNullable(o, ModbusTagAddressResolution.AddressResolvedKey) == true, + // The one authority decides whether the loaded fields are an address: only a blob with the keys is. + // A string-only / invalid-string / address-less blob keeps its keys unwritten until a field is authored. + _authorsKeys = shape == ModbusAddressShape.Keys, _bag = o, }; } @@ -183,21 +235,28 @@ public sealed class ModbusTagConfigModel /// The serialised TagConfig JSON string. public string ToJson() { - // A string-only tag's address fields are not its address (they are load-time defaults): never stamp - // them, or the defaulted region/address would sit beside the string looking authored. - if (!IsStringOnly) + var o = (JsonObject)_bag.DeepClone(); + // Only an authored key address is stamped (see _authorsKeys). A string-only tag's fields are load-time + // defaults, not its address: stamping them would sit HoldingRegisters/0 beside the string, and by the key + // rule that would WIN (review r3 N3). + if (_authorsKeys) { - TagConfigJson.Set(_bag, "region", Region); - TagConfigJson.Set(_bag, "address", Address); - TagConfigJson.Set(_bag, "dataType", DataType); - TagConfigJson.Set(_bag, "byteOrder", ByteOrder); - TagConfigJson.Set(_bag, "bitIndex", BitIndex); - TagConfigJson.Set(_bag, "stringLength", StringLength); + TagConfigJson.Set(o, "region", Region); + TagConfigJson.Set(o, "address", Address); + TagConfigJson.Set(o, "dataType", DataType); + TagConfigJson.Set(o, "byteOrder", ByteOrder); + TagConfigJson.Set(o, "bitIndex", BitIndex); + TagConfigJson.Set(o, "stringLength", StringLength); } - TagConfigJson.Set(_bag, "writable", Writable); - TagConfigJson.Set(_bag, "addressString", AddressString); - TagConfigJson.Set(_bag, ModbusTagDefinitionFactory.AddressResolvedKey, AddressString is not null && AddressResolved ? true : null); - return TagConfigJson.Serialize(_bag); + TagConfigJson.Set(o, "writable", Writable); + // A non-string addressString the model could not load is left as persisted (the driver refuses it, and + // dropping it would change the deploy gate's explanation) unless the address was authored since. + if (AddressString is not null || _authorsKeys) + TagConfigJson.Set(o, ModbusTagAddressResolution.AddressStringKey, AddressString); + // The marker only means something beside keys; a marker on a keyless blob is a stale lie and is dropped. + TagConfigJson.Set(o, ModbusTagAddressResolution.AddressResolvedKey, + AddressString is not null && AddressResolved && _authorsKeys ? true : null); + return TagConfigJson.Serialize(o); } /// Validation hook; returns an error message or null when the model is valid. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs index 00e9ed68..74aa55ba 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusFamilyWiringTests.cs @@ -55,7 +55,8 @@ public sealed class ModbusFamilyWiringTests /// /// Fix round 1 (review m2): persisted region/address are authoritative over an addressString, so a - /// later Family / MelsecSubFamily change can never silently remap a live tag the editor resolved. + /// later Family / MelsecSubFamily change can never silently remap a live tag the editor resolved. (Since + /// fix round 3 the keys win by PRESENCE; the marker in this blob is display state only.) /// [Fact] public void Persisted_resolved_keys_win_over_the_address_string_so_a_family_change_cannot_remap() @@ -69,11 +70,6 @@ public sealed class ModbusFamilyWiringTests } } - /// - /// Fix round 2 (review r1 N1): region/address that were merely STAMPED (an unmarked string beside them — - /// what a model round-trip of a string-only tag used to produce) never win: the string is resolved with - /// the driver's family, so the tag cannot collapse onto HoldingRegisters[0]. - /// /// /// Fix round 3 (review r2 N2): the persisted-shape matrix. Every shape that carries region + address loads /// from them under every family — including the pre-#741 "keys + addressString, no marker" row, which the diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTagAddressResolutionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTagAddressResolutionTests.cs new file mode 100644 index 00000000..37d4d968 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests/ModbusTagAddressResolutionTests.cs @@ -0,0 +1,64 @@ +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Modbus.Tests; + +/// +/// #741 fix round 4 (review r3 N3): is the one authority for which part +/// of a Modbus TagConfig names the address. The cross-consumer agreement matrix lives in AdminUI.Tests +/// (ModbusAddressAuthorityAgreementTests); these pin the resolver's own classification and refusals. +/// +[Trait("Category", "Unit")] +public sealed class ModbusTagAddressResolutionTests +{ + [Theory] + [InlineData("""{"region":"InputRegisters","address":5}""", ModbusAddressShape.Keys)] + [InlineData("""{"region":"HoldingRegisters","address":0,"addressString":"D100"}""", ModbusAddressShape.Keys)] + [InlineData("""{"addressString":"D100","addressResolved":true,"region":"HoldingRegisters","address":100}""", ModbusAddressShape.Keys)] + [InlineData("""{"addressString":"40011:F"}""", ModbusAddressShape.StringOnly)] + [InlineData("""{"addressString":"40011:F","addressResolved":true}""", ModbusAddressShape.StringOnly)] + [InlineData("""{"addressString":"40011:F","region":"InputRegisters"}""", ModbusAddressShape.StringOnly)] + [InlineData("""{"addressString":"ZZ9"}""", ModbusAddressShape.StringOnly)] + [InlineData("""{"addressString":5}""", ModbusAddressShape.InvalidAddressString)] + [InlineData("""{"addressString":" "}""", ModbusAddressShape.NoAddress)] + [InlineData("""{"addressString":null}""", ModbusAddressShape.NoAddress)] + [InlineData("""{}""", ModbusAddressShape.NoAddress)] + [InlineData("""[1]""", ModbusAddressShape.NoAddress)] + [InlineData("""not json""", ModbusAddressShape.NoAddress)] + public void Classify_decides_shape_by_key_presence_never_by_the_marker(string json, ModbusAddressShape shape) + => ModbusTagAddressResolution.Classify(json).ShouldBe(shape); + + [Fact] + public void A_string_only_blob_is_refused_with_a_reason_when_the_family_cannot_parse_it() + { + var r = ModbusTagAddressResolution.Resolve("""{"addressString":"D100"}""", ModbusFamily.Generic, MelsecFamily.Q_L_iQR); + r.IsResolved.ShouldBeFalse(); + r.Shape.ShouldBe(ModbusAddressShape.StringOnly); + r.Refusal.ShouldNotBeNull().ShouldContain("D100"); + + var melsec = ModbusTagAddressResolution.Resolve("""{"addressString":"D100"}""", ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR); + melsec.Address.ShouldNotBeNull().Address.ShouldBe((ushort)100); + } + + [Fact] + public void Keys_are_read_strictly_and_a_bit_region_is_coerced_to_Bool() + { + ModbusTagAddressResolution.Resolve("""{"region":"Holding","address":1}""", ModbusFamily.Generic, MelsecFamily.Q_L_iQR) + .IsResolved.ShouldBeFalse(); + ModbusTagAddressResolution.Resolve("""{"region":"HoldingRegisters","address":70000}""", ModbusFamily.Generic, MelsecFamily.Q_L_iQR) + .IsResolved.ShouldBeFalse(); + var coil = ModbusTagAddressResolution.Resolve("""{"region":"Coils","address":3,"dataType":"Int16"}""", ModbusFamily.Generic, MelsecFamily.Q_L_iQR); + coil.Address.ShouldNotBeNull().DataType.ShouldBe(ModbusDataType.Bool); + } + + [Fact] + public void AddressStringDescribes_matches_only_the_fields_some_family_parses_it_to() + { + ModbusTagAddressResolution.AddressStringDescribes("D100", ModbusRegion.HoldingRegisters, 100, ModbusDataType.Int16, + ModbusByteOrder.BigEndian, 0, 0).ShouldBeTrue(); + ModbusTagAddressResolution.AddressStringDescribes("D100", ModbusRegion.HoldingRegisters, 101, ModbusDataType.Int16, + ModbusByteOrder.BigEndian, 0, 0).ShouldBeFalse(); + ModbusTagAddressResolution.AddressStringDescribes("ZZ9", ModbusRegion.HoldingRegisters, 0, ModbusDataType.Int16, + ModbusByteOrder.BigEndian, 0, 0).ShouldBeFalse(); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusAddressAuthorityAgreementTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusAddressAuthorityAgreementTests.cs new file mode 100644 index 00000000..28148321 --- /dev/null +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusAddressAuthorityAgreementTests.cs @@ -0,0 +1,230 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.AdminUI.Uns; +using ZB.MOM.WW.OtOpcUa.AdminUI.Uns.TagEditors; +using ZB.MOM.WW.OtOpcUa.Configuration.Enums; +using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.Modbus; + +namespace ZB.MOM.WW.OtOpcUa.AdminUI.Tests.Uns; + +/// +/// Review r3 N3 (#741, fix round 4): one authority for a Modbus tag's address. Every persisted +/// TagConfig shape goes through EVERY consumer that decides, displays or rewrites the address. They +/// must all agree with the driver, under every family: +/// +/// the driver's tag reader (ModbusTagDefinitionFactory.FromTagConfig with the driver's +/// family, the exact call ModbusDriver makes at initialize); +/// the deploy gate (EquipmentTagConfigInspector.Inspect); +/// the editor model's load (what its fields claim), save (incl. a Writable toggle) and +/// Validate; +/// the CSV export → import round trip through the real RawTagCsvMapper (the export residual, +/// not the stored blob, is the import base). +/// +/// The expected verdict per family is pinned in the table, so the authority itself cannot drift either. +/// Three rounds converged on the rule one consumer at a time; this matrix is what makes a new divergence +/// fail in one place. +/// +public sealed class ModbusAddressAuthorityAgreementTests +{ + private static readonly (ModbusFamily Family, MelsecFamily Sub, string Label)[] Families = + [ + (ModbusFamily.Generic, MelsecFamily.Q_L_iQR, "Generic"), + (ModbusFamily.DL205, MelsecFamily.Q_L_iQR, "DL205"), + (ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR, "MELSEC-Q"), + (ModbusFamily.MELSEC, MelsecFamily.F_iQF, "MELSEC-F"), + ]; + + /// + /// Shape name, persisted blob, and the expected driver verdict per family in the order of + /// (IR5 = InputRegisters[5], - = refused). + /// + public static TheoryData Shapes => new() + { + { "keys only", """{"region":"InputRegisters","address":5}""", "IR5|IR5|IR5|IR5" }, + { "pre-lane keys + string", """{"region":"InputRegisters","address":5,"dataType":"Int16","addressString":"40011:F"}""", "IR5|IR5|IR5|IR5" }, + { "pre-lane stamped HR/0 + string", + """{"region":"HoldingRegisters","address":0,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0,"addressString":"D100"}""", + "HR0|HR0|HR0|HR0" }, + { "string only", """{"addressString":"40011:F"}""", "HR10|HR10|HR10|HR10" }, + { "string only, family-native", """{"addressString":"D100"}""", "-|-|HR100|HR100" }, + { "string + marker + keys", """{"addressString":"D100","addressResolved":true,"region":"HoldingRegisters","address":100,"dataType":"Int16"}""", "HR100|HR100|HR100|HR100" }, + { "string + marker, no keys", """{"addressString":"40011:F","addressResolved":true}""", "HR10|HR10|HR10|HR10" }, + { "string + marker, no keys, family-native", """{"addressString":"D100","addressResolved":true}""", "-|-|HR100|HR100" }, + { "malformed string", """{"addressString":"ZZ9"}""", "-|-|-|-" }, + { "malformed string + marker", """{"addressString":"ZZ9","addressResolved":true}""", "-|-|-|-" }, + { "malformed string + keys", """{"region":"InputRegisters","address":5,"addressString":"ZZ9"}""", "IR5|IR5|IR5|IR5" }, + { "non-string addressString", """{"addressString":5}""", "-|-|-|-" }, + { "string + one key", """{"addressString":"40011:F","region":"InputRegisters"}""", "HR10|HR10|HR10|HR10" }, + { "no address", """{"writable":true}""", "-|-|-|-" }, + { "bit region, legacy Int16", """{"region":"Coils","address":3,"dataType":"Int16"}""", "CO3|CO3|CO3|CO3" }, + }; + + [Theory] + [MemberData(nameof(Shapes))] + public void Every_consumer_agrees_with_the_driver_on_every_persisted_shape(string shape, string blob, string expected) + { + var want = expected.Split('|'); + var anyServed = want.Any(v => v != "-"); + + // 1. The driver (the authority) — pinned, so the authority cannot drift unnoticed either. + for (var i = 0; i < Families.Length; i++) + { + Verdict(blob, i).ShouldBe(want[i], $"{shape}: driver / {Families[i].Label}"); + } + + // 2. The deploy gate: clean exactly when some family's driver serves the tag. + (EquipmentTagConfigInspector.Inspect(DriverTypeNames.Modbus, blob).Count == 0) + .ShouldBe(anyServed, $"{shape}: deploy gate accepts ⇔ some driver serves it"); + + // 3. Editor load: when the model presents its fields as the address, they ARE the driver's address. + var loaded = ModbusTagConfigModel.FromJson(blob); + if (!loaded.IsStringOnly) + { + for (var i = 0; i < Families.Length; i++) + { + if (want[i] == "-") continue; + FieldsVerdict(loaded).ShouldBe(want[i], $"{shape}: editor fields / {Families[i].Label}"); + } + } + + // 4. Editor save, untouched and after a Writable toggle (the round trip that remapped N1 / N3). + var untouched = ModbusTagConfigModel.FromJson(blob).ToJson(); + var toggled = ModbusTagConfigModel.FromJson(blob); + toggled.Writable = false; + var toggledJson = toggled.ToJson(); + for (var i = 0; i < Families.Length; i++) + { + Verdict(untouched, i).ShouldBe(want[i], $"{shape}: editor save / {Families[i].Label}"); + Verdict(toggledJson, i).ShouldBe(want[i], $"{shape}: editor save after Writable toggle / {Families[i].Label}"); + } + + // 5. Validate: accepts exactly what some family's driver serves. + (ModbusTagConfigModel.FromJson(blob).Validate() is null).ShouldBe(anyServed, $"{shape}: Validate"); + + // 6. CSV export → import through the production mapper (the residual is the import base). + var csv = RawTagCsvMapper.Export(DriverTypeNames.Modbus, + [new RawCsvExportTag(null, "t", "Int16", TagAccessLevel.ReadWrite, false, null, blob)]); + var parsed = RawTagCsvMapper.Parse(DriverTypeNames.Modbus, csv); + parsed.FatalError.ShouldBeNull(); + var row = parsed.Rows.ShouldHaveSingleItem(); + if (!row.Ok) + { + anyServed.ShouldBeFalse($"{shape}: CSV import refused a tag the driver serves ({row.Error})"); + } + else + { + for (var i = 0; i < Families.Length; i++) + { + Verdict(row.ImportRow!.Tag.TagConfig, i).ShouldBe(want[i], $"{shape}: CSV export → import / {Families[i].Label}"); + } + } + } + + private static string Verdict(string json, int familyIndex) + { + var (family, sub, _) = Families[familyIndex]; + return ModbusTagDefinitionFactory.FromTagConfig(json, "T", family, sub, out var def) + ? Short(def.Region) + def.Address + : "-"; + } + + private static string FieldsVerdict(ModbusTagConfigModel m) => Short(m.Region) + m.Address; + + private static string Short(ModbusRegion r) => r switch + { + ModbusRegion.HoldingRegisters => "HR", + ModbusRegion.InputRegisters => "IR", + ModbusRegion.Coils => "CO", + ModbusRegion.DiscreteInputs => "DI", + _ => r.ToString(), + }; + + // ------------------------------------------------------------------ CSV metadata (review r3 m1 / m3) + + private static RawCsvReviewRow RoundTrip(string blob, Func? editCsv = null) + { + var csv = RawTagCsvMapper.Export(DriverTypeNames.Modbus, + [new RawCsvExportTag(null, "t", "Int16", TagAccessLevel.ReadWrite, false, null, blob)]); + if (editCsv is not null) csv = editCsv(csv); + return RawTagCsvMapper.Parse(DriverTypeNames.Modbus, csv).Rows.ShouldHaveSingleItem(); + } + + /// m1: an UNCHANGED export → import of a pinned row keeps its address string and marker. + [Fact] + public void An_unchanged_csv_round_trip_of_a_pinned_row_keeps_its_address_string_and_marker() + { + const string pinned = """{"addressString":"D100","addressResolved":true,"region":"HoldingRegisters","address":100,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0}"""; + var row = RoundTrip(pinned); + row.Ok.ShouldBeTrue(row.Error); + var json = row.ImportRow!.Tag.TagConfig; + json.ShouldContain("\"addressString\":\"D100\""); + json.ShouldContain("\"addressResolved\":true"); + row.TypedOverrodeFallback.ShouldBeEmpty("nothing was overridden"); + ModbusTagConfigModel.FromJson(json).IsPinned.ShouldBeTrue(); + } + + /// m1 positive control: an address cell that CONTRADICTS the string drops it, and the grid says so. + [Fact] + public void A_csv_address_cell_that_contradicts_the_string_drops_it_and_the_review_grid_reports_it() + { + const string pinned = """{"addressString":"D100","addressResolved":true,"region":"HoldingRegisters","address":100,"dataType":"Int16","byteOrder":"BigEndian","bitIndex":0,"stringLength":0}"""; + // Edit the exported Address cell 100 → 101. + var row = RoundTrip(pinned, csv => csv.Replace(",HoldingRegisters,100,", ",HoldingRegisters,101,", StringComparison.Ordinal)); + row.Ok.ShouldBeTrue(row.Error); + var json = row.ImportRow!.Tag.TagConfig; + json.ShouldNotContain("addressString"); + json.ShouldNotContain("addressResolved"); + row.TypedOverrodeFallback.ShouldNotBeEmpty("the dropped string is reported, not silent"); + ModbusTagDefinitionFactory.FromTagConfig(json, "T", ModbusFamily.MELSEC, MelsecFamily.Q_L_iQR, out var def).ShouldBeTrue(); + def.Address.ShouldBe((ushort)101); + } + + /// m3: the n5 case on an UNMARKED string-only row — typed address cells are applied and drop the string. + [Fact] + public void A_typed_csv_address_cell_on_an_unmarked_string_only_row_is_applied_and_drops_the_string() + { + var map = CsvColumnMap.Resolve(DriverTypeNames.Modbus).ShouldNotBeNull(); + var merged = map.ApplyTypedColumns("""{"addressString":"40011:F"}""", + new Dictionary(StringComparer.Ordinal) { ["Region"] = "InputRegisters", ["Address"] = "7" }); + + merged.ShouldNotContain("addressString"); + foreach (var (family, sub, label) in Families) + { + ModbusTagDefinitionFactory.FromTagConfig(merged, "T", family, sub, out var def).ShouldBeTrue(label); + def.Region.ShouldBe(ModbusRegion.InputRegisters, label); + def.Address.ShouldBe((ushort)7, label); + } + } + + /// N3, by name: the marker without keys is string-only for the editor, its save and its CSV export. + [Fact] + public void A_marker_without_keys_is_string_only_and_never_saved_as_HR0() + { + var m = ModbusTagConfigModel.FromJson("""{"addressString":"40011:F","addressResolved":true}"""); + m.IsStringOnly.ShouldBeTrue(); + m.IsPinned.ShouldBeFalse("a marker without keys pins nothing"); + m.Writable = false; + var saved = m.ToJson(); + saved.ShouldNotContain("\"address\":"); + saved.ShouldNotContain("\"region\":"); + CsvColumnMap.Resolve(DriverTypeNames.Modbus)!.ReadTypedColumns(saved).ShouldNotContainKey("Address"); + } + + /// The editor never writes default keys it did not resolve: a Writable toggle on an address-less blob + /// does not author HoldingRegisters[0]; setting a field does. + [Fact] + public void An_address_less_blob_gains_keys_only_when_a_field_is_authored() + { + var m = ModbusTagConfigModel.FromJson("{}"); + m.Writable = false; + m.ToJson().ShouldNotContain("\"region\":"); + m.HasKeyAddress.ShouldBeFalse(); + + m.Address = 0; // an explicit authoring of HR[0] is honoured + m.HasKeyAddress.ShouldBeTrue(); + ModbusTagDefinitionFactory.FromTagConfig(m.ToJson(), "T", out var def).ShouldBeTrue(); + def.Address.ShouldBe((ushort)0); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs index d211ed37..5b124b65 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.AdminUI.Tests/Uns/ModbusTagConfigModelTests.cs @@ -247,11 +247,18 @@ public sealed class ModbusTagConfigModelTests m.Writable = false; var saved = m.ToJson(); - var map = CsvColumnMap.Resolve(ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTypeNames.Modbus).ShouldNotBeNull(); - var cells = map.ReadTypedColumns(PreLaneRow); - cells["Region"].ShouldBe("InputRegisters"); - cells["Address"].ShouldBe("5"); - var reimported = map.ApplyTypedColumns(PreLaneRow, cells); + // The production export → import path (review r3 m2): the import base is the export RESIDUAL, not the row. + var csv = ZB.MOM.WW.OtOpcUa.AdminUI.Uns.RawTagCsvMapper.Export(ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTypeNames.Modbus, + [new ZB.MOM.WW.OtOpcUa.AdminUI.Uns.RawCsvExportTag(null, "t", "Int16", + ZB.MOM.WW.OtOpcUa.Configuration.Enums.TagAccessLevel.ReadWrite, false, null, PreLaneRow)]); + csv.ShouldContain(",InputRegisters,5,"); + var row = ZB.MOM.WW.OtOpcUa.AdminUI.Uns.RawTagCsvMapper.Parse(ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTypeNames.Modbus, csv) + .Rows.ShouldHaveSingleItem(); + row.Ok.ShouldBeTrue(row.Error); + var reimported = row.ImportRow!.Tag.TagConfig; + // The legacy string contradicts the keys under every family, so the import drops it — reported, not silent. + reimported.ShouldNotContain("addressString"); + row.TypedOverrodeFallback.ShouldNotBeEmpty(); foreach (var json in new[] { saved, reimported }) foreach (var family in new[] { ModbusFamily.Generic, ModbusFamily.DL205, ModbusFamily.MELSEC }) @@ -262,10 +269,11 @@ public sealed class ModbusTagConfigModelTests } } - /// Review r2 n5: a typed CSV address cell on a string-only row is an authored address — it is applied - /// and drops the string, mirroring a hand edit in the editor (it used to be silently discarded). + /// Review r2 n5: a typed CSV address cell on a keyless row carrying a (stale) marker is an authored + /// address — it is applied and drops the string it contradicts. The unmarked string-only case is + /// ModbusAddressAuthorityAgreementTests.A_typed_csv_address_cell_on_an_unmarked_string_only_row_… (r3 m3). [Fact] - public void A_typed_csv_address_cell_on_a_string_only_row_is_applied_and_drops_the_string() + public void A_typed_csv_address_cell_on_a_marker_bearing_keyless_row_is_applied_and_drops_the_string() { var map = CsvColumnMap.Resolve(ZB.MOM.WW.OtOpcUa.Core.Abstractions.DriverTypeNames.Modbus).ShouldNotBeNull(); var merged = map.ApplyTypedColumns("""{"addressString":"40011:F","addressResolved":true}""", From dad28d37a2f16ae96702df72aae37126d764de47 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:58:46 -0400 Subject: [PATCH 34/34] fix(alarms): never drop a comment-trail entry on pair divergence (E5 review r1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I-1: a node whose in-memory trail is one short of the stored trail (it missed a command the peer applied, whose entry then replicated in) silently dropped its own next entry — in failover, the survivor's newest ack — because the positional "past MAX(seq)" rule treated it as already stored. SaveAsync now reconciles the last 16 in-memory entries against the stored rows from that position on, by content (Kind/User/Text, timestamps within a 5-minute skew window; matched rows are consumed so repeats count). The peer's copy of the same command is recognised and not re-appended; an unmatched entry is inserted at its own ordinal when free, otherwise appended at MAX(seq)+1 with a Warning — kept, never silent. Bounded read, constant write. m1: MigrateInlineComments copies only — it no longer UPDATEs the condition row (a fresh-HLC re-stamp could beat a newer peer ack under LWW); the next save clears comments_json. m2: it returns unparseable alarm ids and OnReady logs a Warning per id (+ an Information tally). m3/m4: documented in the store remarks (Remove vs lagging peer; in-memory index == durable ordinal). Tests: store-level failover sequence (missed command -> survivor's ack and next entry persisted once, re-save writes 0 rows, one Warning) with the dual-write positive control (peer copy present -> 0 rows, no Warning) and a repeated-identical-comment case; a loopback pair failover test (link down, survivor acks, rejoin -> both entries on both peers, held); migration no-row-rewrite, partial fill and unparseable reporting; OnReady Warning. Both failover tests fail against the round-0 store. Refs #748 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../AlarmConditionStateSchema.cs | 67 +++--- .../Configuration/LocalDbSetup.cs | 29 ++- .../LocalDbAlarmConditionStateStore.cs | 199 +++++++++++++++--- .../AlarmConditionCommentConvergenceTests.cs | 59 ++++++ .../LocalDbSetupTests.cs | 46 +++- .../LocalDbAlarmCommentAuditTests.cs | 172 +++++++++++++-- 6 files changed, 490 insertions(+), 82 deletions(-) diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmConditionStateSchema.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmConditionStateSchema.cs index 1ec6b043..464cb90d 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmConditionStateSchema.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian/AlarmConditionStateSchema.cs @@ -46,7 +46,8 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// O(1) however long the history is. comments_json is kept, always written as /// "[]", so the condition table's shape — and with it the pair's replication digest /// for that table — does not change and an old-shape row stays readable; -/// moves any inline trail across. +/// copies any inline trail across (the alarm's next +/// save clears the column). /// /// /// Why the ordinal, and not a content hash, is the key. Both halves of a redundant pair @@ -55,7 +56,9 @@ namespace ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; /// converged. Keyed by ordinal, the two nodes' copies of entry n share one primary key /// and LWW collapses them to one row — the convergence the whole-row LWW gave before, now per /// entry. Entries go in with INSERT OR IGNORE (), so an -/// entry already held — written locally, or arrived from the peer — is never rewritten. +/// entry already held — written locally, or arrived from the peer — is never rewritten. A +/// node that MISSED a peer command keeps its own colliding entry by appending it after the +/// stored trail, with a Warning (see the store's divergence handling). /// /// public static class AlarmConditionStateSchema @@ -100,8 +103,8 @@ public static class AlarmConditionStateSchema // Acknowledged/Unacknowledged, Confirmed/Unconfirmed, Unshelved/OneShotShelved/TimedShelved), // so they are NOT NULL. comments_json is never null either — it is the literal "[]" since the // trail moved to alarm_condition_comment (an old-shape row may still carry an inline trail - // until MigrateInlineComments or the next save moves it). The audit user/comment/utc columns - // and shelving_expires_utc are NULL when the alarm has no such history yet. updated_at_utc + // until the alarm's next save; MigrateInlineComments copies it without touching this row). + // The audit user/comment/utc columns and shelving_expires_utc are NULL when the alarm has no such history yet. updated_at_utc // carries LastTransitionUtc (there is no dedicated transition column). cmd.CommandText = """ CREATE TABLE IF NOT EXISTS alarm_condition_state ( @@ -134,9 +137,17 @@ public static class AlarmConditionStateSchema cmd.ExecuteNonQuery(); } + /// What one run did. + /// Condition rows for which at least one inline entry was copied across. + /// + /// Alarms whose inline trail is not valid JSON and was left where it is — the caller logs these, + /// so an unmigratable history is visible rather than skipped silently on every boot. + /// + public sealed record InlineCommentMigrationResult(int RowsMoved, IReadOnlyList UnparseableAlarmIds); + /// - /// Moves every condition row's inline comments_json trail into - /// and resets the column to "[]". Idempotent. + /// Copies every condition row's inline comments_json trail into + /// . Idempotent; it never writes the condition row itself. /// /// /// @@ -145,18 +156,26 @@ public static class AlarmConditionStateSchema /// DDL → register → writes rule AlarmSfLegacyMigrator follows). /// /// + /// Copy, don't clear. Resetting comments_json here would be a full-row UPDATE + /// that re-stamps the condition row with a fresh HLC — during a pair's rolling upgrade + /// that stale-but-newly-stamped copy would beat a newer peer state (an ack made on the + /// already-upgraded node) under LWW (review m1). The column is left as it is: the store's + /// load merges an inline trail with the table by ordinal (the table wins), and the + /// engine's next save of that alarm writes "[]" with the alarm's real state. + /// + /// /// Each entry lands at its ordinal in the inline array via , - /// so a re-run — or a row whose trail was already (partly) moved — neither duplicates nor - /// reorders anything, and a second run finds no non-empty inline column and writes - /// nothing. Each row moves in its own transaction together with its column reset, so a - /// crash leaves every row either fully moved or untouched. A row whose inline JSON does - /// not parse is left exactly as it is (the pre-split store could not load it either) - /// rather than failing host start. + /// so a re-run — or a row whose trail was already (partly) copied — neither duplicates nor + /// reorders anything and writes nothing for an ordinal already held. Each row copies in its + /// own transaction, so a crash leaves it fully copied or untouched. A row whose inline JSON + /// does not parse is left exactly as it is (the pre-split store could not load it either) + /// and reported in rather + /// than failing host start. /// /// /// An already-open connection from ILocalDb.CreateConnection(). - /// The number of condition rows whose inline trail was moved. - public static int MigrateInlineComments(SqliteConnection connection) + /// What was copied, and which rows could not be read. + public static InlineCommentMigrationResult MigrateInlineComments(SqliteConnection connection) { ArgumentNullException.ThrowIfNull(connection); @@ -172,6 +191,7 @@ public static class AlarmConditionStateSchema } var migrated = 0; + var unparseable = new List(); foreach (var (alarmId, json) in pending) { IReadOnlyList entries; @@ -181,9 +201,11 @@ public static class AlarmConditionStateSchema } catch (JsonException) { + unparseable.Add(alarmId); continue; } + var inserted = 0; using var tx = connection.BeginTransaction(); for (var seq = 0; seq < entries.Count; seq++) { @@ -196,24 +218,15 @@ public static class AlarmConditionStateSchema insert.Parameters.AddWithValue("@User", entries[seq].User); insert.Parameters.AddWithValue("@Kind", entries[seq].Kind); insert.Parameters.AddWithValue("@Text", entries[seq].Text); - insert.ExecuteNonQuery(); - } - - using (var clear = connection.CreateCommand()) - { - clear.Transaction = tx; - clear.CommandText = - "UPDATE alarm_condition_state SET comments_json = @Empty WHERE scripted_alarm_id = @Id"; - clear.Parameters.AddWithValue("@Empty", EmptyInlineComments); - clear.Parameters.AddWithValue("@Id", alarmId); - clear.ExecuteNonQuery(); + inserted += insert.ExecuteNonQuery(); } tx.Commit(); - migrated++; + if (inserted > 0) + migrated++; } - return migrated; + return new InlineCommentMigrationResult(migrated, unparseable); } /// Formats a comment timestamp for (round-trip "O" text). diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs index c841db21..d3f89979 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Host/Configuration/LocalDbSetup.cs @@ -119,12 +119,33 @@ public static class LocalDbSetup RemoveStaleCapture(db, logger ?? Serilog.Log.Logger); // LAST, and only here. These are the only calls in OnReady that write rows. - // Moves any pre-split inline comment trail (alarm_condition_state.comments_json) into the - // append-only alarm_condition_comment table (review R05:4). Idempotent: a boot after the - // first finds no inline trail and writes nothing. + // Copies any pre-split inline comment trail (alarm_condition_state.comments_json) into the + // append-only alarm_condition_comment table (review R05:4). Idempotent: an ordinal already + // held is never rewritten, and the condition row itself is never touched (the alarm's next + // save clears the column). An unreadable trail is reported, not skipped silently. + AlarmConditionStateSchema.InlineCommentMigrationResult commentMigration; using (var connection = db.CreateConnection()) { - AlarmConditionStateSchema.MigrateInlineComments(connection); + commentMigration = AlarmConditionStateSchema.MigrateInlineComments(connection); + } + + var migrationLog = logger ?? Serilog.Log.Logger; + if (commentMigration.RowsMoved > 0) + { + migrationLog.Information( + "LocalDb: copied the inline comment trail of {AlarmCount} scripted alarm(s) into {Table}.", + commentMigration.RowsMoved, + AlarmConditionStateSchema.CommentsTable); + } + + foreach (var alarmId in commentMigration.UnparseableAlarmIds) + { + migrationLog.Warning( + "LocalDb: scripted alarm {AlarmId} has an unreadable inline comment trail " + + "(alarm_condition_state.comments_json is not valid JSON); its history was NOT moved to {Table} " + + "and the alarm's state will not load until the row is repaired.", + alarmId, + AlarmConditionStateSchema.CommentsTable); } AlarmSfLegacyMigrator.Migrate(db, configuration); diff --git a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/LocalDbAlarmConditionStateStore.cs b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/LocalDbAlarmConditionStateStore.cs index 8437574d..514b7ea9 100644 --- a/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/LocalDbAlarmConditionStateStore.cs +++ b/src/Server/ZB.MOM.WW.OtOpcUa.Runtime/ScriptedAlarms/LocalDbAlarmConditionStateStore.cs @@ -31,13 +31,22 @@ namespace ZB.MOM.WW.OtOpcUa.Runtime.ScriptedAlarms; /// have no columns and default to null on load. /// /// -/// The comment trail is written incrementally (review R05:4). Each save appends only -/// the entries past the highest ordinal already stored for the alarm, so a transition's write -/// — and its replication payload — is constant however long the history is; the condition -/// row's comments_json is always written as "[]". The "already stored" boundary -/// is read from the table inside the save's transaction rather than cached, so the store -/// holds no per-alarm state and an entry the pair peer already wrote at an ordinal is kept, -/// not duplicated (see for why the ordinal is the key). +/// The comment trail is written incrementally (review R05:4). Each save writes only +/// the entries the table does not yet hold, so a transition's write — and its replication +/// payload — is constant however long the history is; the condition row's +/// comments_json is always written as "[]". What is "already stored" is read +/// from the table inside the save's transaction (a bounded read of the trail's tail) rather +/// than cached, so the store holds no per-alarm state. The pair peer's copy of the same +/// command is recognised and not duplicated; a local entry that collides with a peer entry +/// this node never applied is appended after it with a Warning, never dropped — see +/// AppendNewCommentsAsync and, for why the ordinal is the key, +/// . +/// +/// +/// Remove vs. a lagging peer. deletes the whole trail; if the +/// pair peer has not yet applied the same deploy, its next save finds no stored trail and +/// re-inserts every entry, which replicates back. The next deploy removes them again — the +/// same resurrection class the single-row store had, with n+1 rows instead of one (review m3). /// /// /// Old-shape rows load exactly as before. A row still carrying an inline @@ -62,6 +71,20 @@ public sealed class LocalDbAlarmConditionStateStore : IAlarmStateStore private const string SelectCommentColumns = "scripted_alarm_id, seq, timestamp_utc, user_name, kind, text"; + /// + /// How many trailing in-memory entries each save reconciles against the stored trail by + /// content. The engine appends at most one entry per save, so this comfortably covers a few + /// missed commands while keeping each save's read bounded. + /// + private const int MatchWindow = 16; + + /// + /// How far apart two nodes' timestamps for the SAME command may be and still count as one + /// entry: command fan-out latency plus inter-node clock skew. Beyond it both copies are kept + /// (the safe direction — a duplicate, never a loss). + /// + private static readonly TimeSpan SameCommandSkew = TimeSpan.FromMinutes(5); + private readonly ILocalDb _db; private readonly ILogger _logger; @@ -128,32 +151,7 @@ public sealed class LocalDbAlarmConditionStateStore : IAlarmStateStore await using var tx = await _db.BeginTransactionAsync(ct).ConfigureAwait(false); - // Append only what lies past the highest ordinal already stored. An indexed MAX over the - // (scripted_alarm_id, seq) primary key — O(log n), and a read, so it costs no replication. - var maxSeq = (await tx.QueryAsync( - "SELECT COALESCE(MAX(seq), -1) FROM alarm_condition_comment WHERE scripted_alarm_id = @Id", - static r => r.GetInt64(0), - new { Id = state.AlarmId }, - ct).ConfigureAwait(false))[0]; - - for (var seq = (int)Math.Max(maxSeq + 1, 0); seq < state.Comments.Count; seq++) - { - var c = state.Comments[seq]; - await tx.ExecuteAsync( - AlarmConditionStateSchema.InsertCommentSql, - new - { - Id = state.AlarmId, - Seq = seq, - // AlarmComment.TimestampUtc must be DateTimeKind.Utc for a Utc round-trip; the - // engine always creates comments with Utc kind. - Ts = AlarmConditionStateSchema.FormatTimestamp(c.TimestampUtc), - User = c.User, - Kind = c.Kind, - Text = c.Text, - }, - ct).ConfigureAwait(false); - } + await AppendNewCommentsAsync(tx, state, ct).ConfigureAwait(false); // A single unconditional upsert keyed on the alarm identity. Convergence across the pair is // last-writer-wins over the primary key, handled by the LocalDb replication HLC — no @@ -209,6 +207,141 @@ public sealed class LocalDbAlarmConditionStateStore : IAlarmStateStore _logger.LogTrace("Persisted alarm-condition state for {AlarmId}", state.AlarmId); } + /// + /// Persists the trail entries holds that the table does not, writing + /// a bounded number of rows however long the trail is. + /// + /// + /// + /// Normal case. The in-memory trail and the stored trail agree position by + /// position; the only new entries are past MAX(seq) and land at their own ordinal. + /// + /// + /// Divergence (review I-1). If this node missed a command its peer applied, the + /// peer's entry arrives in this table at an ordinal this node's in-memory trail does not + /// know about, and this node's NEXT entry has an in-memory index at or below + /// MAX(seq). Positional "already stored" would silently drop it — the survivor's + /// newest operator action, in the failover case. So the tail of the in-memory trail + /// (the last entries) is matched against the stored rows from + /// that position on, by content: same Kind, User and Text, timestamps within + /// (each node stamps the same command with its own clock). + /// A matched stored row is consumed, so repeated identical entries are counted, not + /// merged. An entry with no match is new: at its own ordinal when that is past the stored + /// trail, otherwise appended after it (MAX(seq)+1) with a Warning — kept, never + /// dropped, and never silent. + /// + /// + /// Why this cannot double-count. Both pair nodes apply every command; the second + /// node to save finds the first's copy among the stored rows (same Kind/User/Text, close + /// timestamp) and consumes it instead of appending. A node's own earlier divergence-append + /// is found the same way, so a re-save writes nothing. What it cannot tell apart: a peer + /// entry this node missed and a byte-identical entry this node later records within the + /// skew window — that pair is kept once. A clock skew between the nodes beyond the window + /// errs the other way (both copies kept), never towards loss. True union semantics need an + /// originator-assigned entry id on the alarm command (follow-up F3). + /// + /// + /// ⚠️ The in-memory index is the durable ordinal. Trimming + /// in the engine without carrying a persisted + /// base offset would make every later save look "already stored" — do not bound that list + /// without changing this method. + /// + /// + private async Task AppendNewCommentsAsync(ILocalDbTransaction tx, AlarmConditionState state, CancellationToken ct) + { + var comments = state.Comments; + var count = comments.Count; + + // An indexed MAX over the (scripted_alarm_id, seq) primary key — a read, so no replication cost. + var max = (await tx.QueryAsync( + "SELECT COALESCE(MAX(seq), -1) FROM alarm_condition_comment WHERE scripted_alarm_id = @Id", + static r => r.GetInt64(0), + new { Id = state.AlarmId }, + ct).ConfigureAwait(false))[0]; + + var windowStart = Math.Max(0, count - MatchWindow); + + // Entries below the window that lie past the stored trail (a first save of a long trail, an + // old-shape row's lazy move): plain appends at their own ordinal — nothing is stored there. + for (var seq = (int)Math.Max(max + 1, 0); seq < windowStart; seq++) + { + await InsertCommentAsync(tx, state.AlarmId, seq, comments[seq], ct).ConfigureAwait(false); + max = seq; + } + + if (windowStart >= count) return; + + // The stored rows the window can match: ordinal >= windowStart. Bounded by the window plus the + // number of commands this node missed. + var stored = max >= windowStart + ? await tx.QueryAsync( + $"SELECT {SelectCommentColumns} FROM alarm_condition_comment WHERE scripted_alarm_id = @Id AND seq >= @Lo ORDER BY seq", + MapComment, + new { Id = state.AlarmId, Lo = windowStart }, + ct).ConfigureAwait(false) + : []; + var consumed = new bool[stored.Count]; + + for (var i = windowStart; i < count; i++) + { + var entry = comments[i]; + var match = -1; + for (var j = 0; j < stored.Count; j++) + { + if (!consumed[j] && IsSameEntry(stored[j].Comment, entry)) + { + match = j; + break; + } + } + + if (match >= 0) + { + consumed[match] = true; + continue; + } + + if (i > max) + { + await InsertCommentAsync(tx, state.AlarmId, i, entry, ct).ConfigureAwait(false); + max = i; + continue; + } + + var target = max + 1; + await InsertCommentAsync(tx, state.AlarmId, target, entry, ct).ConfigureAwait(false); + max = target; + _logger.LogWarning( + "Alarm {AlarmId}: comment trail diverged from the pair peer — the {Kind} entry by {User} at in-memory " + + "position {Position} collides with a stored entry this node never applied (a missed command). " + + "Kept: appended at ordinal {Ordinal}.", + state.AlarmId, entry.Kind, entry.User, i, target); + } + } + + private static bool IsSameEntry(AlarmComment stored, AlarmComment local) => + string.Equals(stored.Kind, local.Kind, StringComparison.Ordinal) + && string.Equals(stored.User, local.User, StringComparison.Ordinal) + && string.Equals(stored.Text, local.Text, StringComparison.Ordinal) + && (stored.TimestampUtc - local.TimestampUtc).Duration() <= SameCommandSkew; + + private static Task InsertCommentAsync( + ILocalDbTransaction tx, string alarmId, long seq, AlarmComment c, CancellationToken ct) => + tx.ExecuteAsync( + AlarmConditionStateSchema.InsertCommentSql, + new + { + Id = alarmId, + Seq = seq, + // AlarmComment.TimestampUtc must be DateTimeKind.Utc for a Utc round-trip; the engine + // always creates comments with Utc kind. + Ts = AlarmConditionStateSchema.FormatTimestamp(c.TimestampUtc), + User = c.User, + Kind = c.Kind, + Text = c.Text, + }, + ct); + /// public async Task RemoveAsync(string alarmId, CancellationToken ct) { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmConditionCommentConvergenceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmConditionCommentConvergenceTests.cs index 26b1fe14..8a6a7e62 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmConditionCommentConvergenceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDb/AlarmConditionCommentConvergenceTests.cs @@ -70,6 +70,65 @@ public sealed class AlarmConditionCommentConvergenceTests (await storeA.LoadAsync("eq/alarm-1", Ct))!.Comments.ShouldBe(appended.Comments); } + [Fact] + public async Task Failover_AfterAMissedCommand_BothTheMissedEntryAndTheSurvivorsNextEntry_ReachBothPeers() + { + // Review I-1, end to end over the real pair: + // 1. a shared trail on both nodes; + // 2. an AddComment reaches only node A (B's command delivery failed) — A's entry replicates + // into B's table, but B's engine never applied it, so B's IN-MEMORY trail is one short; + // 3. A goes away (link down) and the operator acks on the survivor B — B's in-memory Ack + // collides with A's entry at the same ordinal; + // 4. the pair rejoins. + // Both entries must survive on both peers. Before the fix B's Ack was silently dropped. + await using var pair = new LocalDbPairHarness(); + await pair.StartAsync(); + + var shared = AlarmConditionState.Fresh("eq/alarm-fo", T0) with + { + Active = AlarmActiveState.Active, + Acked = AlarmAckedState.Unacknowledged, + }; + for (var i = 0; i < 3; i++) + shared = Part9StateMachine.ApplyAddComment(shared, "jane", $"note {i}", T0.AddMinutes(i)).State; + await Store(pair.A).SaveAsync(shared, Ct); + await LocalDbPairHarness.WaitUntilAsync( + async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 3, + "node B to hold the shared trail"); + + var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State; + await Store(pair.A).SaveAsync(onlyOnA, Ct); + await LocalDbPairHarness.WaitUntilAsync( + async () => await CommentCountAsync(pair.B, "eq/alarm-fo") == 4, + "node A's entry to replicate into node B's table"); + + await pair.StopPassiveAsync(); + + // B's engine still holds the 3-entry trail it applied; the operator acks there. + var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State; + await Store(pair.B).SaveAsync(ackOnB, Ct); + + await pair.RestartPairAsync(); + + await LocalDbPairHarness.WaitUntilAsync( + async () => await CommentCountAsync(pair.A, "eq/alarm-fo") == 5 + && await CommentCountAsync(pair.B, "eq/alarm-fo") == 5, + "both nodes to hold the shared trail + A's missed-by-B entry + B's ack"); + + var onA = (await Store(pair.A).LoadAsync("eq/alarm-fo", Ct))!; + var onB = (await Store(pair.B).LoadAsync("eq/alarm-fo", Ct))!; + onA.Comments.ShouldBe(onB.Comments); + onA.Comments[3].ShouldBe(onlyOnA.Comments[3]); + onA.Comments[4].ShouldBe(ackOnB.Comments[3]); + + // Held, not merely reached: nothing late duplicates either entry. + (await LocalDbPairHarness.HeldWithinAsync( + async () => await CommentCountAsync(pair.A, "eq/alarm-fo") != 5 + || await CommentCountAsync(pair.B, "eq/alarm-fo") != 5, + TimeSpan.FromSeconds(2))) + .ShouldBeFalse(); + } + [Fact] public async Task BothNodesRecordingTheSameEntry_ConvergeToOneEntry_NotTwo() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs index 280ac35a..06351576 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Host.IntegrationTests/LocalDbSetupTests.cs @@ -250,8 +250,11 @@ public sealed class LocalDbSetupTests : IDisposable (await ScalarAsync(db, "SELECT COUNT(*) FROM alarm_condition_comment WHERE scripted_alarm_id = 'eq/alarm-legacy'")) .ShouldBe(2); - (await ScalarAsync(db, "SELECT COUNT(*) FROM alarm_condition_state WHERE comments_json <> '[]'")) - .ShouldBe(0); + // The condition row itself is NOT rewritten (review m1): a migration UPDATE would re-stamp it + // with a fresh HLC and could beat a newer peer state under LWW. Only the seeding INSERT is in + // the oplog for it; the alarm's next save clears the inline column. + (await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_condition_state'")) + .ShouldBe(1); // Captured: the migration ran after RegisterReplicated, so the moved entries reach the peer. (await ScalarAsync(db, "SELECT COUNT(*) FROM __localdb_oplog WHERE table_name = 'alarm_condition_comment'")) .ShouldBe(2); @@ -397,6 +400,45 @@ public sealed class LocalDbSetupTests : IDisposable sink.Events.Count(e => e.Level == LogEventLevel.Information).ShouldBe(1); } + [Fact] + public async Task OnReady_WarnsOncePerAlarmWhoseInlineCommentTrailIsUnreadable() + { + // Review m2: an unmigratable history must be visible, not skipped silently on every boot. + var first = Boot(); + await first.ExecuteAsync( + """ + INSERT INTO alarm_condition_state + (scripted_alarm_id, enabled_state, acked_state, confirmed_state, shelving_state, + comments_json, updated_at_utc) + VALUES (@Id, 'Enabled', 'Acknowledged', 'Confirmed', 'Unshelved', @Json, '2026-06-10T10:05:00.0000000Z') + """, + new { Id = "eq/alarm-bad", Json = "{not json" }, + TestContext.Current.CancellationToken); + await first.ExecuteAsync( + """ + INSERT INTO alarm_condition_state + (scripted_alarm_id, enabled_state, acked_state, confirmed_state, shelving_state, + comments_json, updated_at_utc) + VALUES (@Id, 'Enabled', 'Acknowledged', 'Confirmed', 'Unshelved', @Json, '2026-06-10T10:05:00.0000000Z') + """, + new + { + Id = "eq/alarm-good", + Json = """[{"timestampUtc":"2026-06-10T10:00:00Z","user":"jane","kind":"AddComment","text":"a"}]""", + }, + TestContext.Current.CancellationToken); + + var sink = new CollectingSink(); + Boot(replication: null, logger: BuildLogger(sink)); + + var warnings = sink.Events.Where(e => e.Level == LogEventLevel.Warning).ToList(); + warnings.Count.ShouldBe(1); + warnings[0].RenderMessage().ShouldContain("eq/alarm-bad"); + // Positive control: the readable row DID move, and said so. + sink.Events.Count(e => e.Level == LogEventLevel.Information && e.RenderMessage().Contains("copied")) + .ShouldBe(1); + } + [Fact] public void DisablingReplication_SaysNothingWhenThereWasNothingToClean() { diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/LocalDbAlarmCommentAuditTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/LocalDbAlarmCommentAuditTests.cs index 47438ceb..e6c422e9 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/LocalDbAlarmCommentAuditTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.Runtime.Tests/ScriptedAlarms/LocalDbAlarmCommentAuditTests.cs @@ -252,7 +252,12 @@ public sealed class LocalDbAlarmCommentAuditTests : IDisposable var rewritten = original with { Comments = original.Comments.SetItem(1, new AlarmComment(T0, "eve", "AddComment", "forged")) }; await store.SaveAsync(rewritten, Ct); - (await store.LoadAsync("alarm-ao", Ct))!.Comments.ShouldBe(original.Comments); + // The stored entry at ordinal 1 is untouched; the unrecognised local entry is kept by + // appending it (never dropped), not by overwriting history. + var loaded = (await store.LoadAsync("alarm-ao", Ct))!.Comments; + loaded.Take(3).ShouldBe(original.Comments); + loaded.Count.ShouldBe(4); + loaded[3].Text.ShouldBe("forged"); } // ---- back-compat + migration ----------------------------------------------------------- @@ -306,19 +311,27 @@ public sealed class LocalDbAlarmCommentAuditTests : IDisposable } [Fact] - public async Task Migration_MovesInlineCommentsToTheAuditTable_AndClearsTheInlineColumn() + public async Task Migration_CopiesInlineCommentsToTheAuditTable_AndNeverRewritesTheConditionRow() { await InsertOldShapeRowAsync("alarm-m1"); await InsertOldShapeRowAsync("alarm-m2"); await InsertOldShapeRowAsync("alarm-empty", "[]"); - int migrated; + var before = await OplogHighWaterAsync(); + AlarmConditionStateSchema.InlineCommentMigrationResult result; using (var connection = _db.CreateConnection()) - migrated = AlarmConditionStateSchema.MigrateInlineComments(connection); + result = AlarmConditionStateSchema.MigrateInlineComments(connection); - migrated.ShouldBe(2); + result.RowsMoved.ShouldBe(2); + result.UnparseableAlarmIds.ShouldBeEmpty(); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(6); - (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_state WHERE comments_json <> '[]'")).ShouldBe(0); + + // Review m1: the migration must not touch the condition row. A full-row UPDATE re-stamps it + // with a fresh HLC and can revert a newer peer state (e.g. an ack) under LWW. The next save + // (the engine's boot save) resets the column instead. + (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(0); + (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(6, + "positive control: the copied entries were captured"); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m1", Ct), "alarm-m1"); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-m2", Ct), "alarm-m2"); @@ -330,11 +343,11 @@ public sealed class LocalDbAlarmCommentAuditTests : IDisposable { await InsertOldShapeRowAsync("alarm-idem"); using (var connection = _db.CreateConnection()) - AlarmConditionStateSchema.MigrateInlineComments(connection).ShouldBe(1, "positive control"); + AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1, "positive control"); var before = await OplogHighWaterAsync(); using (var connection = _db.CreateConnection()) - AlarmConditionStateSchema.MigrateInlineComments(connection).ShouldBe(0); + AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(0); (await OplogHighWaterAsync()).ShouldBe(before, "a re-run must write nothing"); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3); @@ -342,24 +355,151 @@ public sealed class LocalDbAlarmCommentAuditTests : IDisposable } [Fact] - public async Task Migration_OverAPartiallyMigratedRow_DoesNotDuplicateOrReorder() + public async Task Migration_OverAPartiallyMigratedRow_FillsOnlyTheMissingOrdinals() { - // An inline row reappearing after its comments were already moved (e.g. a restored backup): - // the ordinals already held are kept, nothing is duplicated. + // Only ordinal 0 made it across (e.g. a restored backup): the run fills 1 and 2, keeps 0, + // duplicates nothing. await InsertOldShapeRowAsync("alarm-partial"); - using (var connection = _db.CreateConnection()) - AlarmConditionStateSchema.MigrateInlineComments(connection); await _db.ExecuteAsync( - "UPDATE alarm_condition_state SET comments_json = @Json WHERE scripted_alarm_id = 'alarm-partial'", - new { Json = OldShapeCommentsJson }, Ct); + AlarmConditionStateSchema.InsertCommentSql, + new + { + Id = "alarm-partial", Seq = 0, + Ts = AlarmConditionStateSchema.FormatTimestamp(OldShapeComments[0].TimestampUtc), + User = OldShapeComments[0].User, Kind = OldShapeComments[0].Kind, Text = OldShapeComments[0].Text, + }, + Ct); using (var connection = _db.CreateConnection()) - AlarmConditionStateSchema.MigrateInlineComments(connection).ShouldBe(1); + AlarmConditionStateSchema.MigrateInlineComments(connection).RowsMoved.ShouldBe(1); (await ScalarAsync("SELECT COUNT(*) FROM alarm_condition_comment")).ShouldBe(3); ShouldBeTheOldShapeState(await NewStore().LoadAsync("alarm-partial", Ct), "alarm-partial"); } + [Fact] + public async Task Migration_ReportsAnUnparseableInlineTrail_AndLeavesItAlone() + { + // Review m2: an unreadable trail must surface (OnReady logs a Warning per id), not be + // skipped silently on every boot. + await InsertOldShapeRowAsync("alarm-bad", "{not json"); + await InsertOldShapeRowAsync("alarm-good"); + + AlarmConditionStateSchema.InlineCommentMigrationResult result; + using (var connection = _db.CreateConnection()) + result = AlarmConditionStateSchema.MigrateInlineComments(connection); + + result.UnparseableAlarmIds.ShouldBe(["alarm-bad"]); + result.RowsMoved.ShouldBe(1, "positive control: the good row still moved"); + (await _db.QueryAsync( + "SELECT comments_json FROM alarm_condition_state WHERE scripted_alarm_id = 'alarm-bad'", + r => r.GetString(0), null, Ct))[0] + .ShouldBe("{not json"); + } + + // ---- divergence / failover (review I-1) ------------------------------------------------- + + /// Captures log entries so a divergence Warning can be asserted, not inferred. + private sealed class CapturingLogger : Microsoft.Extensions.Logging.ILogger + { + public List<(Microsoft.Extensions.Logging.LogLevel Level, string Message)> Entries { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(Microsoft.Extensions.Logging.LogLevel logLevel) => true; + + public void Log( + Microsoft.Extensions.Logging.LogLevel logLevel, Microsoft.Extensions.Logging.EventId eventId, + TState state, Exception? exception, Func formatter) + { + lock (Entries) Entries.Add((logLevel, formatter(state, exception))); + } + + public int Warnings => Entries.Count(e => e.Level == Microsoft.Extensions.Logging.LogLevel.Warning); + } + + [Fact] + public async Task Failover_TheSurvivorsNextEntry_IsPersisted_AfterItMissedACommandThePeerApplied() + { + // The I-1 sequence on node B's database: + // 1. a shared trail of 5 entries; + // 2. an AddComment reaches only node A (B's command delivery failed once) and A's row at + // ordinal 5 replicates into B's table — B's IN-MEMORY trail still has 5 entries; + // 3. A dies and the operator acks on B: B's in-memory Ack sits at index 5, colliding with + // A's entry at ordinal 5. + // The Ack must be persisted (and say so), not silently treated as "already stored". + var shared = Unacked("alarm-fo", Comments(5)); + var a = NewStore(); + await a.SaveAsync(shared, Ct); + var onlyOnA = Part9StateMachine.ApplyAddComment(shared, "jane", "seen on A only", T0.AddHours(1)).State; + await a.SaveAsync(onlyOnA, Ct); + + var log = new CapturingLogger(); + var b = new LocalDbAlarmConditionStateStore(_db, log); + var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "acked on B", T0.AddHours(2)).State; + await b.SaveAsync(ackOnB, Ct); + + var loaded = (await NewStore().LoadAsync("alarm-fo", Ct))!; + loaded.Comments.Count.ShouldBe(7); + loaded.Comments[5].ShouldBe(onlyOnA.Comments[5]); + loaded.Comments[6].ShouldBe(ackOnB.Comments[5]); + log.Warnings.ShouldBe(1, "a divergence is never silent"); + + // Re-saving with no new entry must not append the Ack a second time (its own earlier + // divergence-append shifted it off its in-memory index). + var before = await OplogHighWaterAsync(); + await b.SaveAsync(ackOnB with { LastTransitionUtc = T0.AddHours(3) }, Ct); + (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0); + (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.StateTable)).ShouldBe(1, + "positive control: the re-save itself was captured"); + + // And B's following entry (still one behind in memory) is persisted too, exactly once. + var confirmOnB = Part9StateMachine.ApplyAddComment(ackOnB, "bob", "follow-up on B", T0.AddHours(4)).State; + await b.SaveAsync(confirmOnB, Ct); + await b.SaveAsync(confirmOnB, Ct); + var after = (await NewStore().LoadAsync("alarm-fo", Ct))!; + after.Comments.Count.ShouldBe(8); + after.Comments[7].ShouldBe(confirmOnB.Comments[6]); + } + + [Fact] + public async Task PeersCopyOfTheSameCommand_IsNotAppendedAgain_AndLogsNothing() + { + // The positive control for the divergence path: in normal operation both nodes apply every + // command, each with its own clock. When the peer's copy is already at the ordinal, the + // local copy is the same entry and must NOT be appended — otherwise the fix would double + // every entry on a healthy pair. + var shared = Unacked("alarm-dual", Comments(5)); + await NewStore().SaveAsync(shared, Ct); + var ackOnA = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1)).State; + await NewStore().SaveAsync(ackOnA, Ct); + + var log = new CapturingLogger(); + var b = new LocalDbAlarmConditionStateStore(_db, log); + var ackOnB = Part9StateMachine.ApplyAcknowledge(shared, "bob", "on it", T0.AddHours(1).AddMilliseconds(40)).State; + var before = await OplogHighWaterAsync(); + await b.SaveAsync(ackOnB, Ct); + + (await OplogEntriesAboveAsync(before, AlarmConditionStateSchema.CommentsTable)).ShouldBe(0); + (await NewStore().LoadAsync("alarm-dual", Ct))!.Comments.Count.ShouldBe(6); + log.Warnings.ShouldBe(0); + } + + [Fact] + public async Task ARepeatedIdenticalComment_IsKept_AsItsOwnEntry() + { + // Matching ignores the timestamp only within the clock-skew window; an operator repeating the + // same text later is a new entry, not a duplicate of the earlier one. + var store = NewStore(); + var s = Unacked("alarm-repeat", Comments(2)); + s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(1)).State; + await store.SaveAsync(s, Ct); + s = Part9StateMachine.ApplyAddComment(s, "amy", "still high", T0.AddHours(2)).State; + await store.SaveAsync(s, Ct); + + (await NewStore().LoadAsync("alarm-repeat", Ct))!.Comments.ShouldBe(s.Comments); + } + [Fact] public async Task SavingALoadedOldShapeState_MovesItsCommentsAndAppendsTheNewOne() {