From 01efd3917732eee6a30367369adcf53abb32429c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:22:17 -0400 Subject: [PATCH 01/20] fix(mqtt): retry failed SUBSCRIBEs via a pending set; throw on no-verdict subscribe (R06:M-1, M-11) A failed SUBSCRIBE (refusal or no broker verdict) used to drop its topic from every retry set while SubscribeAsync returned a live handle, leaving tags dark behind a Healthy connection until a redeploy. Failed topics now join a pending set retried by the next SubscribeAsync and by every reconnect (not counted toward the reconnect's total-failure verdict, so an ACL-denied topic cannot reconnect-loop). A no-verdict subscribe degrades its refs then throws so the actor's reconcile re-subscribes. A subscribe with no transport attached is recorded pending, never established (M-11). MiniBroker now answers SUBSCRIBE. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 15 +- .../MqttSubscriptionManager.cs | 131 ++++++++++---- .../MiniBroker.cs | 52 ++++++ .../MqttDriverSessionLifecycleTests.cs | 53 +++++- .../MqttSubscriptionManagerTests.cs | 163 +++++++++++++++++- 5 files changed, 374 insertions(+), 40 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index f8aef796..8400ab42 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -408,9 +408,18 @@ value. `MqttConnection` therefore raises `MqttConnectRejectedException`: code. (`ServerMoved` is permanent, `UseAnotherServer` is temporary — per the MQTT 5 spec.) A reconnect that completes without re-subscribing would be a healthy-looking connection that receives -nothing forever, so `OnReconnectedAsync` **throws** when zero filters are granted, tearing the session -down rather than swallowing it. A SUBACK failure degrades just the affected RawPaths to -`BadCommunicationError`. +nothing forever, so `OnReconnectedAsync` **throws** when none of the previously live filters is granted +(or the SUBSCRIBE gets no verdict), tearing the session down rather than swallowing it. A SUBACK failure +degrades just the affected RawPaths to `BadCommunicationError`. + +**A failed SUBSCRIBE is retried, never forgotten (R06:M-1).** Every topic whose SUBSCRIBE was refused or got +no verdict joins a *pending* set that the next `SubscribeAsync` and every reconnect re-request; a grant moves +it to the live set, and a redeploy that drops its tags drops it. Pending topics do not count toward the +reconnect's total-failure verdict, so one ACL-denied topic cannot reconnect-loop the driver. A plain-mode +`SubscribeAsync` whose SUBSCRIBE got **no verdict at all** (transport throw, SUBSCRIBE deadline) degrades its +refs and then **throws**, so `DriverInstanceActor` holds no handle and its health-tick reconcile re-subscribes; +a SUBACK *refusal* still returns a handle (the broker answered). A subscribe issued with no transport attached +records its topics as pending, never as established (M-11). There are **two** kinds of reconnect and only one of them fires that callback. A drop the connection's own supervisor recovers keeps the same `MqttConnection`, so `Reconnected` → `OnReconnectedAsync` diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 92c45051..9e4d8f64 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -113,6 +113,14 @@ public sealed class MqttSubscriptionManager /// Topics already SUBSCRIBEd on the live session — the dedupe set and the re-subscribe set. private readonly ConcurrentDictionary _subscribedTopics = new(StringComparer.Ordinal); + /// + /// Topics a subscribe asked for whose SUBSCRIBE failed (refused, or no verdict at all). Retried by + /// every later and by ; moved to + /// when granted. Without it a failed topic left every retry set and + /// stayed dark behind a Healthy connection until a redeploy. + /// + private readonly ConcurrentDictionary _pendingTopics = new(StringComparer.Ordinal); + /// RawPaths whose extraction failure has already been logged loudly once. private readonly ConcurrentDictionary _warnedRefs = new(StringComparer.Ordinal); @@ -212,8 +220,8 @@ public sealed class MqttSubscriptionManager /// The manager does not own the connection's lifetime and never detaches — the driver disposes /// both together. Call once per connection, and before the first /// : a subscribe issued with no transport attached records its - /// intent (and warns) but sends nothing to a broker, and would only be established if the - /// connection later dropped and recovered. + /// topics as pending (and warns) but sends nothing to a broker; the next subscribe or + /// reconnect on an attached session retries them. /// /// /// Binding to a session forgets the previous session's established filters — the same @@ -328,6 +336,14 @@ public sealed class MqttSubscriptionManager } } + foreach (var topic in _pendingTopics.Keys) + { + if (!table.ByFilter.ContainsKey(topic)) + { + _pendingTopics.TryRemove(topic, out _); + } + } + foreach (var rawPath in _handleByRawPath.Keys) { if (!table.ByRawPath.ContainsKey(rawPath)) @@ -356,12 +372,13 @@ public sealed class MqttSubscriptionManager /// opening SUBSCRIBE itself; only re-establishes it later. /// /// - /// Never throws, never hangs. A reference that is not an authored tag is recorded as - /// BadNodeIdUnknown; a SUBACK rejection, transport failure or subscribe deadline - /// degrades exactly the affected references to BadCommunicationError. Either way a - /// handle is returned — the OPC UA server's subscribe call must not fail because one topic - /// is ACL-denied. Recovery for a transient failure comes from the reconnect path; - /// a persistent broker-side refusal is visible as a Bad status on the affected nodes. + /// Never hangs; throws only when the broker gave no verdict. A reference that is not an + /// authored tag is recorded as BadNodeIdUnknown. A SUBACK rejection degrades exactly the + /// affected references to BadCommunicationError and still returns a handle — the OPC UA + /// server's subscribe must not fail because one topic is ACL-denied. A transport failure or + /// subscribe deadline degrades the same way and then throws, so the host holds no handle + /// and its reconcile re-subscribes. Every failed topic also joins a pending set that the next + /// subscribe and every reconnect retry. /// /// /// The RawPath references to subscribe. @@ -414,12 +431,43 @@ public sealed class MqttSubscriptionManager // One SUBSCRIBE per distinct topic, and only for topics not already live: re-SUBSCRIBEing an // established filter is legal but pointless, and it re-triggers the retained seed. - var pending = defs.Where(d => !_subscribedTopics.ContainsKey(d.Topic)).ToList(); - await EstablishAsync(BuildFilters(pending, _defaultQos), cancellationToken).ConfigureAwait(false); + // Topics an EARLIER subscribe failed to establish ride along: a redeploy's bulk re-subscribe is then a + // retry of a refusal the broker may since have lifted, instead of a no-op that leaves them dark. + var authored = _authored; + var pending = defs.Where(d => !_subscribedTopics.ContainsKey(d.Topic)) + .Concat(PendingDefinitions(authored)) + .ToList(); + var outcome = await EstablishAsync(BuildFilters(pending, _defaultQos), cancellationToken).ConfigureAwait(false); + if (outcome.TransportFailure is { } failure) + { + // No verdict at all (transport throw / SUBSCRIBE deadline): THROW, so the caller holds no handle + // and DriverInstanceActor's health-tick reconcile re-subscribes. Returning a handle here was the + // silent-death case — the actor thought the subscription live and never retried. The refs already + // carry BadCommunicationError (raised under this handle above), so the throw costs no visibility. + _refsByHandle.TryRemove(handle, out _); + foreach (var def in defs) + { + _handleByRawPath.TryRemove(new KeyValuePair(def.Name, handle)); + } + + throw new InvalidOperationException( + $"MQTT driver '{_driverId}': SUBSCRIBE for {pending.Count} tag(s) got no broker verdict " + + $"({failure.GetType().Name}); the subscription will be retried.", + failure); + } + + // A SUBACK refusal (an ACL denial, say) does NOT throw: the broker answered, the refused topics are + // Bad + pending, and failing the whole subscribe would take every other topic in it dark too. return handle; } + /// The authored definitions behind every still-authored pending topic. + private IEnumerable PendingDefinitions(AuthoredTable authored) + => _pendingTopics.Keys + .Select(topic => authored.ByFilter.TryGetValue(topic, out var defs) ? defs : []) + .SelectMany(defs => defs); + /// /// Stops attributing notifications to . References another live /// subscription also covers keep publishing. @@ -498,8 +546,13 @@ public sealed class MqttSubscriptionManager .SelectMany(defs => defs) .ToList(); - var filters = BuildFilters(live, _defaultQos); - if (filters.Count == 0) + // Pending topics (a failed earlier SUBSCRIBE) are retried here too — a reconnect re-subscribing + // only the granted set is how a failed topic stayed dark forever. They do NOT count toward the + // total-failure verdict below: re-refusing a topic that was never live loses nothing, and throwing + // for it would turn one ACL-denied topic into a reconnect loop for the whole driver. + var pendingDefs = PendingDefinitions(authored).ToList(); + var filters = BuildFilters(live.Concat(pendingDefs), _defaultQos); + if (live.Count == 0 && pendingDefs.Count == 0) { // Silence here is how the wildcard defect hid: no exception, no log, no SUBSCRIBE. Live // topics resolving to zero filters re-establishes nothing, which is the total-failure case @@ -519,13 +572,17 @@ public sealed class MqttSubscriptionManager return; } - var granted = await EstablishAsync(filters, cancellationToken).ConfigureAwait(false); - if (granted == 0) + var outcome = await EstablishAsync(filters, cancellationToken).ConfigureAwait(false); + var liveTopics = live.Select(d => d.Topic).Distinct(StringComparer.Ordinal).ToList(); + var liveLost = outcome.TransportFailure is not null + || (subscribed.Count > 0 && (liveTopics.Count == 0 || !liveTopics.Any(outcome.Granted.Contains))); + if (liveLost) { throw new InvalidOperationException( $"MQTT driver '{_driverId}': no subscription could be re-established after reconnect " + $"({filters.Count} filter(s) requested); tearing the session down rather than serving a " - + "connection that receives nothing."); + + "connection that receives nothing.", + outcome.TransportFailure); } } @@ -633,35 +690,35 @@ public sealed class MqttSubscriptionManager /// /// The filters to establish; an empty list is a no-op. /// Cancellation for the round-trip. - /// How many filters the broker granted. - private async Task EstablishAsync( + /// The granted topics, plus the transport failure when the broker gave no verdict at all. + private async Task EstablishAsync( IReadOnlyList filters, CancellationToken cancellationToken) { if (filters.Count == 0) { - return 0; + return EstablishOutcome.Nothing; } var transport = _transport; if (transport is null) { - // No transport: the intent is recorded so the reconnect path establishes it, but NOTHING - // reaches a broker now. Loud, because outside tests this means the composition order is - // wrong — AttachTo must run before the OPC UA server's first subscribe, or these topics - // stay unestablished until the connection happens to drop and recover. + // No transport: NOTHING reaches a broker, so nothing is recorded as subscribed — recording it + // was a silent "subscribed, receives nothing" if the AttachTo-before-subscribe ordering ever + // changed. The topics wait in the pending set, which the next subscribe and every reconnect + // retry. Loud, because outside tests this means the composition order is wrong. foreach (var filter in filters) { - _subscribedTopics[filter.Topic] = 0; + _pendingTopics[filter.Topic] = 0; } _logger?.LogWarning( - "MQTT driver '{DriverId}': {FilterCount} subscription(s) recorded with no connection attached; " - + "nothing was sent to a broker. Call AttachTo before subscribing.", + "MQTT driver '{DriverId}': {FilterCount} subscription(s) requested with no connection attached; " + + "nothing was sent to a broker and they stay pending. Call AttachTo before subscribing.", _driverId, filters.Count); - return filters.Count; + return EstablishOutcome.Nothing; } IReadOnlyList outcomes; @@ -681,16 +738,17 @@ public sealed class MqttSubscriptionManager DegradeTopic(filter.Topic, ex.GetType().Name); } - return 0; + return new EstablishOutcome(new HashSet(StringComparer.Ordinal), ex); } - var granted = 0; + var granted = new HashSet(StringComparer.Ordinal); foreach (var outcome in outcomes) { if (outcome.Granted) { _subscribedTopics[outcome.Topic] = 0; - granted++; + _pendingTopics.TryRemove(outcome.Topic, out _); + granted.Add(outcome.Topic); } else { @@ -703,7 +761,16 @@ public sealed class MqttSubscriptionManager } } - return granted; + return new EstablishOutcome(granted, null); + } + + /// One SUBSCRIBE round-trip's verdict. + /// The filters the broker granted. + /// Set when the call produced no verdict at all (throw / deadline). + private sealed record EstablishOutcome(IReadOnlySet Granted, Exception? TransportFailure) + { + public static readonly EstablishOutcome Nothing = + new(new HashSet(StringComparer.Ordinal), null); } /// Marks every authored reference on as unreachable. @@ -722,6 +789,7 @@ public sealed class MqttSubscriptionManager if (!_authored.ByFilter.TryGetValue(topic, out var defs)) { + _pendingTopics.TryRemove(topic, out _); _logger?.LogWarning( "MQTT driver '{DriverId}': cannot degrade tags for unestablished filter '{Topic}' ({Reason}) — " + "no authored tag resolves to it.", @@ -731,6 +799,9 @@ public sealed class MqttSubscriptionManager return; } + // Still authored, so it must be retried — see _pendingTopics. + _pendingTopics[topic] = 0; + var now = DateTime.UtcNow; var snapshot = new DataValueSnapshot(null, StatusBadCommunicationError, null, now); foreach (var def in defs) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs index 7950a240..77d0b762 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs @@ -59,6 +59,54 @@ internal sealed class MiniBroker : IDisposable /// public byte ConnAckReturnCode { get; set; } = ReturnCodeAccepted; + /// + /// When true (the default) a SUBSCRIBE is answered with a SUBACK granting each filter's requested + /// QoS. False is a broker that accepts SUBSCRIBE and never answers — the subscribe-deadline shape. + /// + public bool AnswerSubscribe { get; set; } = true; + + /// + /// Builds the SUBACK for one SUBSCRIBE packet read whole into , or null when + /// the packet is not parseable from one read (the fake handles one packet per read, like CONNECT). + /// + private static byte[]? TryBuildSubAck(byte[] buffer, int read) + { + // Fixed header: type byte, then the variable-length "remaining length". + var at = 1; + var remaining = 0; + var multiplier = 1; + byte digit; + do + { + if (at >= read) return null; + digit = buffer[at++]; + remaining += (digit & 0x7F) * multiplier; + multiplier *= 128; + } + while ((digit & 0x80) != 0); + + var end = at + remaining; + if (end > read || remaining < 2) return null; + + var packetIdHigh = buffer[at]; + var packetIdLow = buffer[at + 1]; + at += 2; + + var grants = new List(); + while (at + 2 <= end) + { + var length = (buffer[at] << 8) | buffer[at + 1]; + at += 2 + length; + if (at >= end) return null; + grants.Add((byte)(buffer[at] & 0x03)); + at++; + } + + var ack = new List { 0x90, (byte)(2 + grants.Count), packetIdHigh, packetIdLow }; + ack.AddRange(grants); + return [.. ack]; + } + /// Total TCP connections accepted since construction — counts reconnect attempts. public int AcceptedCount => Volatile.Read(ref _acceptedCount); @@ -163,6 +211,10 @@ internal sealed class MiniBroker : IDisposable { await stream.WriteAsync(PingResp, _cts.Token); } + else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(buffer, read) is { } subAck) + { + await stream.WriteAsync(subAck, _cts.Token); + } } } catch (Exception) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index ca4e667e..947fb081 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -13,9 +13,9 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; /// ( / ). /// /// -/// Plain mode deliberately: it is the only mode whose initialize needs no SUBSCRIBE, and -/// answers CONNECT and PINGREQ and nothing else — so a Sparkplug driver -/// would park on its group SUBSCRIBE until the deadline and prove nothing about connections. +/// Plain mode deliberately: it is the only mode whose initialize needs no SUBSCRIBE. +/// answers CONNECT, SUBSCRIBE (unless told not to) and PINGREQ, and never +/// PUBLISHes, so ingest is driven through Subscriptions.HandleMessage. /// [Trait("Category", "Unit")] public sealed class MqttDriverSessionLifecycleTests @@ -122,7 +122,7 @@ public sealed class MqttDriverSessionLifecycleTests /// /// /// Ingest is fed through Subscriptions.HandleMessage, the method MQTTnet's dispatcher - /// calls, because answers CONNECT and PINGREQ and nothing else: it + /// calls, because answers CONNECT, SUBSCRIBE and PINGREQ only: it /// can drop a session (which is what this test needs from it) but cannot PUBLISH. /// /// @@ -174,6 +174,51 @@ public sealed class MqttDriverSessionLifecycleTests recovered.Value.ShouldBe(21.5); } + /// + /// R06:M-1, end to end. A SUBSCRIBE the broker never answers used to come back from + /// as a live handle while the topic left every retry set, so + /// DriverInstanceActor never re-subscribed and the tag stayed dark behind a Healthy connection. + /// It must throw (the actor's reconcile then retries), and the retry against a broker that answers + /// must establish and deliver — the positive control. + /// + [Trait("Category", "SlowUnit")] + [Fact] + public async Task ASubscribeTheBrokerNeverAnswers_Throws_AndTheRetryEstablishes() + { + const string rawPath = "Plant/Mqtt/dev1/Temp"; + const string topic = "factory/oven/temp"; + + using var broker = new MiniBroker { AnswerSubscribe = false }; + var options = BrokerOptions(broker.Port) with + { + RawTags = + [ + new RawTagEntry( + rawPath, + $$"""{"topic":"{{topic}}","payloadFormat":"Scalar","dataType":"Float64"}""", + WriteIdempotent: false), + ], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + + await Should.ThrowAsync(() => + driver.SubscribeAsync([rawPath], TimeSpan.Zero, TestContext.Current.CancellationToken)); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "the connection itself is fine — that was the trap"); + + broker.AnswerSubscribe = true; + var handle = await driver.SubscribeAsync([rawPath], TimeSpan.Zero, TestContext.Current.CancellationToken); + + var raised = new List(); + driver.OnDataChange += (_, e) => { lock (raised) raised.Add(e); }; + driver.Subscriptions.HandleMessage(topic, "7"u8.ToArray(), retained: false); + lock (raised) + { + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.StatusCode.ShouldBe(0u); + } + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs index 558d4477..632b0925 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs @@ -538,17 +538,171 @@ public sealed class MqttSubscriptionManagerTests mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); } + /// + /// A SUBSCRIBE that never got a verdict (transport throw / deadline) degrades its refs AND throws, + /// so DriverInstanceActor holds no handle and its health-tick reconcile re-subscribes. It + /// used to return a live-looking handle with the topic dropped from every retry set — dark behind a + /// Healthy connection until a redeploy. + /// [Fact] - public async Task SubscribeAsync_TransportThrows_DegradesRefsToBad_DoesNotThrow() + public async Task SubscribeAsync_TransportThrows_DegradesRefsToBad_AndThrows() { var transport = new FakeTransport { Throw = new InvalidOperationException("broker gone") }; var mgr = NewManager(transport); mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); - await Should.NotThrowAsync(() => + await Should.ThrowAsync(() => mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); + // The Bad verdict still reached the subscriber before the throw. + raised.ShouldContain(e => e.FullReference == OvenTempPath && e.Snapshot.StatusCode == BadCommunicationError); + } + + /// + /// The reconcile path: once the actor re-subscribes after the throw, the topic IS re-SUBSCRIBEd and + /// delivery resumes on the new handle — the positive control for the throw above. + /// + [Fact] + public async Task SubscribeAsync_AfterATransportThrow_TheRetrySubscribesAndDelivers() + { + var transport = new FakeTransport { Throw = new InvalidOperationException("broker gone") }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await Should.ThrowAsync(() => + mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); + + transport.Throw = null; + var handle = await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); + mgr.HandleMessage(OvenTopic, "21.5"u8.ToArray(), retained: false); + + transport.Calls.Count.ShouldBe(2); + transport.Calls[1].Select(f => f.Topic).ShouldBe([OvenTopic]); + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.Value.ShouldBe(21.5); + } + + /// + /// A topic whose first SUBSCRIBE was refused stays in a pending set the reconnect re-subscribe + /// covers. The reconnect used to re-SUBSCRIBE only the granted set, so it was never retried. + /// + [Fact] + public async Task OnReconnectedAsync_RetriesATopicWhoseFirstSubscribeWasRefused() + { + var transport = new FakeTransport { Reject = _ => true }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + var handle = await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Reject = _ => false; + await mgr.OnReconnectedAsync(TestContext.Current.CancellationToken); + + transport.Calls.Count.ShouldBe(2); + transport.Calls[1].Select(f => f.Topic).ShouldBe([OvenTopic]); + + var raised = new List(); + mgr.OnDataChange += (_, e) => raised.Add(e); + mgr.HandleMessage(OvenTopic, "3"u8.ToArray(), retained: false); + raised.ShouldHaveSingleItem().SubscriptionHandle.ShouldBe(handle); + raised[0].Snapshot.StatusCode.ShouldBe(Good); + } + + /// + /// A persistently ACL-denied topic must not turn every reconnect into a teardown loop: a pending + /// topic refused again is not "re-established nothing" — nothing that was live was lost. Both a + /// mixed set (the granted topic still re-subscribes) and a pending-only set are covered. + /// + [Fact] + public async Task OnReconnectedAsync_PendingTopicsRefusedAgain_DoesNotThrow() + { + var transport = new FakeTransport { Reject = t => t == "f/oven/press" }; + var mgr = NewManager(transport); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenTempPath, OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + await Should.NotThrowAsync(() => mgr.OnReconnectedAsync(TestContext.Current.CancellationToken)); + transport.Calls[1].Select(f => f.Topic).OrderBy(t => t, StringComparer.Ordinal) + .ShouldBe(["f/oven/press", OvenTopic]); + + var lone = new FakeTransport { Reject = _ => true }; + var mgr2 = NewManager(lone); + mgr2.Register([Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr2.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + await Should.NotThrowAsync(() => mgr2.OnReconnectedAsync(TestContext.Current.CancellationToken)); + lone.Calls.Count.ShouldBe(2); // it WAS retried + } + + /// + /// A later subscribe on the same session (a redeploy's bulk re-subscribe) also retries the pending + /// set, so a broker ACL fixed while connected recovers without waiting for a reconnect. + /// + [Fact] + public async Task SubscribeAsync_RetriesPendingTopicsFromAnEarlierRefusal() + { + var transport = new FakeTransport { Reject = t => t == "f/oven/press" }; + var mgr = NewManager(transport); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Reject = _ => false; + await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + transport.Calls[1].Select(f => f.Topic).OrderBy(t => t, StringComparer.Ordinal) + .ShouldBe(["f/oven/press", OvenTopic]); + } + + /// + /// R06:M-11. A subscribe issued before any transport is attached reaches no broker, so it must + /// not be recorded as established. It used to be — "subscribed, receives nothing" — and the first + /// reconnect then had nothing to re-establish. Now it stays pending, and the first reconnect on the + /// attached session attempts it: the Bad verdict from the unconnected session is the proof. + /// + [Fact] + public async Task SubscribeAsync_WithNoTransport_StaysPending_AndTheFirstReconnectAttemptsIt() + { + var mgr = NewManager(transport: null); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(0x80320000u); // BadWaitingForInitialData + + await using var attached = new MqttConnection( + new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false }, "mqtt-1"); + mgr.AttachTo(attached); + + // The unconnected session gives no verdict, so the reconnect throws — and it ATTEMPTED the topic. + await Should.ThrowAsync(() => mgr.OnReconnectedAsync(TestContext.Current.CancellationToken)); + mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); + } + + /// + /// Register drops a pending topic whose tags the redeploy removed — the same housekeeping the + /// granted set gets — so a deleted tag's denied topic is not retried forever. + /// + [Fact] + public async Task Register_DroppingATag_RemovesItsTopicFromThePendingSet() + { + var transport = new FakeTransport { Reject = _ => true }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.SubscribeAsync([OvenPressPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + await mgr.OnReconnectedAsync(TestContext.Current.CancellationToken); + + transport.Calls.Count.ShouldBe(1); // nothing left to retry } // --------------------------------------------------------------------------------- @@ -939,7 +1093,10 @@ public sealed class MqttSubscriptionManagerTests new MqttDriverOptions { Host = "127.0.0.1", Port = 1, UseTls = false }, "mqtt-1"); mgr.AttachTo(rebound); - await mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + // The unconnected session gives no verdict, so the subscribe now THROWS (M-1) — the throw is a + // second proof the SUBSCRIBE was attempted against the new session. + await Should.ThrowAsync(() => + mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe( BadCommunicationError, From 43f45bb9781e1aaad731f1d2ba61d90ea898becc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:26:55 -0400 Subject: [PATCH 02/20] fix(mqtt): dispose an unpublished connection and stamp lifecycle generations (R06:M-3, M-10) A refused initial connect (or a failed Sparkplug establish) now disposes the MqttConnection it built. Every lifecycle operation takes a generation (initialize inside the gate; shutdown/dispose before waiting for it): a wedged initialize that completes after a timed-out ShutdownAsync's ungated teardown releases its session instead of publishing it, and a superseded operation's health/host-state writes are refused. TeardownAsync captures every field before disposing so an ungated teardown never reaches into a later initialize's state. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../MqttDriver.cs | 139 +++++++++++++++--- .../MqttDriverSessionLifecycleTests.cs | 69 +++++++++ 2 files changed, 186 insertions(+), 22 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs index b8e92621..26cdeab6 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs @@ -208,6 +208,15 @@ public sealed class MqttDriver private SparkplugPrimaryHost? _primaryHost; private DriverHealth _health = new(DriverState.Unknown, null, null); + + /// Guards the generation bump against the generation-checked health write. + private readonly object _healthLock = new(); + + /// + /// Bumped by every lifecycle operation (initialize inside the gate; shutdown and dispose before + /// waiting for it). The stamp that keeps a superseded operation from publishing state. + /// + private long _lifecycleGeneration; private HostState _hostState = HostState.Unknown; private DateTime _hostStateChangedUtc = DateTime.UtcNow; private int _disposed; @@ -289,6 +298,9 @@ public sealed class MqttDriver /// internal Func? BeforeConnectHookForTests { get; set; } + /// Test seam: observes every an initialize builds. Null in production. + internal Action? ConnectionCreatedForTests { get; set; } + // ---- IDriver: lifecycle ---- /// @@ -387,6 +399,11 @@ public sealed class MqttDriver // watchdog's reconnect) is measured against, reporting a driver that was delivering a moment ago // as one that has never delivered. var lastMessage = GetHealth().LastSuccessfulRead; + + // Taken BEFORE the gate wait: a wedged initialize holding the gate is superseded by this call, so + // when it finally completes it disposes the session it built instead of publishing it behind the + // ungated teardown below (R06:M-10), and its late health write is refused. + var generation = BeginLifecycleOperation(); if (await _lifecycleGate.WaitAsync(TimeSpan.FromSeconds(_options.ConnectTimeoutSeconds), cancellationToken) .ConfigureAwait(false)) { @@ -408,8 +425,10 @@ public sealed class MqttDriver await TeardownAsync().ConfigureAwait(false); } - WriteHealth(new DriverHealth(DriverState.Unknown, lastMessage, null)); - TransitionHost(HostState.Unknown); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, lastMessage, null))) + { + TransitionHost(HostState.Unknown); + } } /// @@ -911,6 +930,9 @@ public sealed class MqttDriver return; } + // Supersedes any in-flight lifecycle operation, exactly as ShutdownAsync does. + BeginLifecycleOperation(); + if (await _lifecycleGate.WaitAsync(TimeSpan.FromSeconds(_options.ConnectTimeoutSeconds)) .ConfigureAwait(false)) { @@ -940,7 +962,12 @@ public sealed class MqttDriver /// The gate-held body of . private async Task InitializeCoreAsync(string driverConfigJson, CancellationToken cancellationToken) { - WriteHealth(new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); + // Taken INSIDE the gate (the caller holds it), so a second initialize queued behind this one does + // not supersede it; only a ShutdownAsync / DisposeAsync — which take theirs before waiting — can. + var generation = BeginLifecycleOperation(); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); + MqttConnection? built = null; + var published = false; try { // Tear down whatever this driver already holds BEFORE building a new session. Both fields @@ -965,6 +992,8 @@ public sealed class MqttDriver RegisterAuthoredTags(_options); var connection = new MqttConnection(_options, _driverInstanceId, _logger); + built = connection; + ConnectionCreatedForTests?.Invoke(connection); // MUST precede the connect: this wires message delivery AND the reconnect re-subscribe. // The handler is passed through unwrapped on purpose — its throw-on-total-failure is what @@ -992,10 +1021,13 @@ public sealed class MqttDriver // disposing what we just built, never leaking it. if (Volatile.Read(ref _disposed) != 0) { - await connection.DisposeAsync().ConfigureAwait(false); throw new ObjectDisposedException(nameof(MqttDriver)); } + // The same race against a ShutdownAsync whose bounded gate wait expired: it has already torn + // down ungated, so publishing now would leave a live session no later shutdown can see. + ThrowIfSuperseded(generation); + // Sparkplug subscribes ONCE, here, rather than on the OPC UA server's first subscribe: the // driver must see birth certificates whether or not anything is subscribed yet, because a // birth is where every alias and datatype comes from. This also issues the late-join @@ -1008,28 +1040,44 @@ public sealed class MqttDriver // MqttConnection's own remarks describe having already fixed once. if (_sparkplug is { } sparkplug) { - try - { - await sparkplug.EstablishAsync(cancellationToken).ConfigureAwait(false); - } - catch - { - await connection.DisposeAsync().ConfigureAwait(false); - throw; - } + await sparkplug.EstablishAsync(cancellationToken).ConfigureAwait(false); } + ThrowIfSuperseded(generation); _connection = connection; + published = true; + + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Healthy, connection.LastMessageUtc, null))) + { + TransitionHost(HostState.Running); + } - WriteHealth(new DriverHealth(DriverState.Healthy, connection.LastMessageUtc, null)); - TransitionHost(HostState.Running); StartHostProbe(); await StartPrimaryHostAsync(cancellationToken).ConfigureAwait(false); + + // Re-checked after publishing: a shutdown whose gate wait expired between the check above and + // here captured whatever it found and left these fields for us. We still hold the gate, so no + // newer initialize can have published anything — everything in them is ours to release. + if (!IsCurrentLifecycle(generation)) + { + await TeardownAsync().ConfigureAwait(false); + ThrowIfSuperseded(generation); + } } catch (Exception ex) { - WriteHealth(new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, ex.Message)); - TransitionHost(HostState.Faulted); + // R06:M-3: a connection built but never published is disposed here — a refused connect, a + // failed Sparkplug establish, a supersede. The published one belongs to TeardownAsync. + if (!published && built is not null) + { + await built.DisposeAsync().ConfigureAwait(false); + } + + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, ex.Message))) + { + TransitionHost(HostState.Faulted); + } + throw; } } @@ -1380,24 +1428,28 @@ public sealed class MqttDriver /// Shared teardown for , and the session rebuild. private async Task TeardownAsync() { + // Every field is CAPTURED before anything is disposed. A shutdown that timed out on the gate runs + // this ungated; capturing up front means it only ever releases what existed when it started, and + // never reaches into fields a later initialize publishes while the slow disposals below run. + var primaryHost = Interlocked.Exchange(ref _primaryHost, null); + var probe = Interlocked.Exchange(ref _hostProbeCts, null); + var connection = Interlocked.Exchange(ref _connection, null); + // Stand the host claim down FIRST, while the broker is still reachable and before the ingest // session goes. Its explicit retained OFFLINE is what makes the transition immediate for edge // nodes; leaving it to the Will costs them up to 1.5 keep-alive periods of believing a host // that has already stopped is still online. - var primaryHost = Interlocked.Exchange(ref _primaryHost, null); if (primaryHost is not null) { await primaryHost.DisposeAsync().ConfigureAwait(false); } - var probe = Interlocked.Exchange(ref _hostProbeCts, null); if (probe is not null) { await probe.CancelAsync().ConfigureAwait(false); probe.Dispose(); } - var connection = Interlocked.Exchange(ref _connection, null); if (connection is not null) { await connection.DisposeAsync().ConfigureAwait(false); @@ -1479,6 +1531,49 @@ public sealed class MqttDriver /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); - /// Barrier-protected publish of a new health snapshot. - private void WriteHealth(DriverHealth value) => Volatile.Write(ref _health, value); + /// + /// Starts a lifecycle operation and returns its generation. Every later operation supersedes it: + /// its health / host-state writes are refused and an initialize stops before publishing a session. + /// + private long BeginLifecycleOperation() + { + lock (_healthLock) + { + return ++_lifecycleGeneration; + } + } + + /// Whether is still the newest lifecycle operation. + private bool IsCurrentLifecycle(long generation) => Interlocked.Read(ref _lifecycleGeneration) == generation; + + /// Throws when a newer lifecycle operation has superseded . + private void ThrowIfSuperseded(long generation) + { + if (!IsCurrentLifecycle(generation)) + { + throw new OperationCanceledException( + $"MQTT driver '{_driverInstanceId}': this initialize was superseded by a later shutdown/dispose; " + + "the session it built has been released."); + } + } + + /// + /// Publishes a health snapshot only while is the newest lifecycle + /// operation — the check and the write are atomic against , so a + /// superseded initialize or shutdown can never overwrite a newer one's state. + /// + /// when the write was applied. + private bool WriteHealthIfCurrent(long generation, DriverHealth value) + { + lock (_healthLock) + { + if (_lifecycleGeneration != generation) + { + return false; + } + + Volatile.Write(ref _health, value); + return true; + } + } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 947fb081..76d12cdd 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -219,6 +219,75 @@ public sealed class MqttDriverSessionLifecycleTests } } + /// + /// R06:M-3. A refused initial connect used to rethrow without disposing the + /// (and its IMqttClient) it had just built. Control: a + /// successful initialize leaves its connection live, so the seam is not reporting "disposed" for + /// everything. + /// + [Fact] + public async Task InitializeAsync_RefusedConnect_DisposesTheConnectionItBuilt() + { + using var broker = new MiniBroker { ConnAckReturnCode = MiniBroker.ReturnCodeNotAuthorized }; + var options = BrokerOptions(broker.Port); + await using var driver = new MqttDriver(options, "d", null); + var built = new List(); + driver.ConnectionCreatedForTests = c => built.Add(c); + + await Should.ThrowAsync(() => driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken)); + + built.ShouldHaveSingleItem().State.ShouldBe(MqttConnectionState.Disposed); + + broker.ConnAckReturnCode = MiniBroker.ReturnCodeAccepted; + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + built.Count.ShouldBe(2); + built[1].State.ShouldBe(MqttConnectionState.Connected); + } + + /// + /// R06:M-10 + the lifecycle-generation rule. A shutdown that times out on the lifecycle gate + /// tears down ungated; the wedged initialize that held the gate must then NOT publish its freshly + /// connected session behind that teardown, and its late health write must not overwrite the + /// shutdown's. The positive controls: the wedged init really did reach the broker (AcceptedCount), + /// and a later initialize connects and reports Healthy. + /// + [Trait("Category", "SlowUnit")] + [Fact] + public async Task AWedgedInit_CompletingAfterATimedOutShutdown_PublishesNothing() + { + using var broker = new MiniBroker(); + var options = BrokerOptions(broker.Port) with { ConnectTimeoutSeconds = 1 }; + await using var driver = new MqttDriver(options, "d", null); + + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + driver.BeforeConnectHookForTests = async _ => + { + entered.TrySetResult(); + await release.Task; + }; + + var initialize = Task.Run(() => driver.InitializeAsync(Json(options), CancellationToken.None)); + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + // Times out on the gate (1 s) and tears down ungated. + await driver.ShutdownAsync(TestContext.Current.CancellationToken); + driver.GetHealth().State.ShouldBe(DriverState.Unknown); + + driver.BeforeConnectHookForTests = null; + release.SetResult(); + await Should.ThrowAsync(() => initialize); + + broker.AcceptedCount.ShouldBe(1, "the wedged initialize did connect — the control for the absence below"); + await WaitForAsync(() => broker.LiveConnections == 0, "the superseded session to be torn down"); + driver.GetHealth().State.ShouldBe(DriverState.Unknown, "a superseded init must not overwrite newer health"); + driver.GetHostStatuses().ShouldHaveSingleItem().State.ShouldBe(HostState.Unknown); + + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + await WaitForAsync(() => broker.LiveConnections == 1, "the fresh session"); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); From 57ffbf79e5609738337a77b4906483445c1796c1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:28:33 -0400 Subject: [PATCH 03/20] fix(mqtt): prune LastValueCache to the authored set on Register (R06:M-2) Both ingest paths now drop cache entries whose RawPath is no longer authored when a deploy re-registers, so a removed tag's value and every non-authored subscribe reference's BadNodeIdUnknown marker no longer accumulate for the driver's lifetime. Still-authored tags keep their observed values. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../LastValueCache.cs | 24 +++++++++++++++++ .../MqttSubscriptionManager.cs | 5 ++++ .../Sparkplug/SparkplugIngestor.cs | 5 ++++ .../MqttSubscriptionManagerTests.cs | 27 +++++++++++++++++++ .../SparkplugIngestorTests.cs | 22 +++++++++++++++ 5 files changed, 83 insertions(+) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs index 7df9dd0e..5ba310bf 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/LastValueCache.cs @@ -45,6 +45,30 @@ public sealed class LastValueCache _values[rawPath] = snapshot; } + /// How many RawPaths currently hold an entry. Diagnostics and the bounded-growth tests. + internal int Count => _values.Count; + + /// + /// Drops every entry whose RawPath rejects — called by each ingest path's + /// Register with "is this still an authored tag", so the cache is bounded by the CURRENT + /// authored set rather than by the union of every RawPath (and every non-authored reference ever + /// subscribed) across the driver's lifetime (R06:M-2). + /// + /// Returns for a RawPath whose entry must survive. + /// How many entries were dropped. + public int RetainOnly(Func keep) + { + ArgumentNullException.ThrowIfNull(keep); + var dropped = 0; + foreach (var rawPath in _values.Keys) + { + if (!keep(rawPath) && _values.TryRemove(rawPath, out _)) + dropped++; + } + + return dropped; + } + /// /// Read the last observed value for . Never throws: a /// null/empty or a RawPath never observed both return a diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 9e4d8f64..1504328a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -325,6 +325,11 @@ public sealed class MqttSubscriptionManager _authored = table; _warnedRefs.Clear(); + // Bound the shared last-value cache by the CURRENT authored set (R06:M-2): a dropped tag's value + // and every non-authored reference's BadNodeIdUnknown marker would otherwise live forever. A + // still-authored tag keeps its value, so a redeploy never blanks what has been observed. + Values.RetainOnly(table.ByRawPath.ContainsKey); + // Drop state belonging to tags this deploy removed. Without this, a topic whose last tag was // deleted stays in _subscribedTopics and is dutifully re-SUBSCRIBEd on every future reconnect, // forever — and its RawPaths keep a handle mapping that can never fire again. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs index cd368eae..5d63e1b7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/Sparkplug/SparkplugIngestor.cs @@ -416,6 +416,11 @@ public sealed class SparkplugIngestor _authored = table; _warned.Clear(); + // Bound the shared last-value cache by the CURRENT authored set (R06:M-2): a dropped tag's value + // and every non-authored reference's BadNodeIdUnknown marker would otherwise live forever. A + // still-authored tag keeps its value, so a redeploy never blanks what has been observed. + Values.RetainOnly(table.ByRawPath.ContainsKey); + // Drop state belonging to edge nodes this deploy removed. Without this a tracker (and its // rebirth debounce slot) for a node nobody reads survives every future redeploy. foreach (var key in _trackers.Keys) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs index 632b0925..906c6da4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs @@ -664,6 +664,33 @@ public sealed class MqttSubscriptionManagerTests .ShouldBe(["f/oven/press", OvenTopic]); } + /// + /// R06:M-2. Register prunes the cache to the authored set — an unauthored reference's + /// BadNodeIdUnknown marker and a dropped tag's value go. Control: a still-authored tag keeps its + /// value across the re-register (a redeploy must not blank observed values). + /// + [Fact] + public async Task Register_PrunesCacheEntriesForReferencesNoLongerAuthored() + { + var mgr = NewManager(new FakeTransport()); + mgr.Register( + [ + Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}"""), + Tag(OvenPressPath, """{"topic":"f/oven/press","payloadFormat":"Scalar","dataType":"Float64"}"""), + ]); + await mgr.SubscribeAsync([OvenTempPath, OvenPressPath, "Plant/Mqtt/broker1/Nope"], TimeSpan.Zero, TestContext.Current.CancellationToken); + mgr.HandleMessage(OvenTopic, "1"u8.ToArray(), retained: false); + mgr.HandleMessage("f/oven/press", "2"u8.ToArray(), retained: false); + mgr.Values.Read("Plant/Mqtt/broker1/Nope").StatusCode.ShouldBe(BadNodeIdUnknown); + + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + + mgr.Values.Read(OvenTempPath).Value.ShouldBe(1.0); + mgr.Values.Read(OvenPressPath).StatusCode.ShouldBe(0x80320000u); + mgr.Values.Read("Plant/Mqtt/broker1/Nope").StatusCode.ShouldBe(0x80320000u); + mgr.Values.Count.ShouldBe(1); + } + /// /// R06:M-11. A subscribe issued before any transport is attached reaches no broker, so it must /// not be recorded as established. It used to be — "subscribed, receives nothing" — and the first diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs index 1bd3b830..43c44103 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/SparkplugIngestorTests.cs @@ -852,6 +852,28 @@ public sealed class SparkplugIngestorTests ing.TryResolve(TempPath, out _).ShouldBeFalse(); } + /// + /// R06:M-2. Register prunes the shared cache to the authored set: an unauthored reference's + /// BadNodeIdUnknown marker and a dropped tag's value go, so the cache cannot grow with every RawPath + /// and stray reference ever seen. Control: a still-authored tag's value survives the re-register. + /// + [Fact] + public async Task Register_PrunesCacheEntriesForReferencesNoLongerAuthored() + { + var ing = NewIngestor(); + await ing.SubscribeAsync(["Plant/Mqtt/spb/NotAThing"], TimeSpan.Zero, TestContext.Current.CancellationToken); + var kept = new DataValueSnapshot(1.0f, 0u, null, DateTime.UtcNow); + ing.Values.Update(TempPath, kept); + ing.Values.Update(PressPath, kept); + + ing.Register([Tag(TempPath, Blob(Group, Node, Device, "Temperature", "Float32"))]); + + ing.Values.Read(TempPath).ShouldBe(kept); + ing.Values.Read(PressPath).StatusCode.ShouldBe(0x80320000u); // dropped tag: no stale value + ing.Values.Read("Plant/Mqtt/spb/NotAThing").StatusCode.ShouldBe(0x80320000u); + ing.Values.Count.ShouldBe(1); + } + [Fact] public async Task SubscribeAsync_UnauthoredReference_ReportsBadNodeIdUnknown() { From d67396cc6d7eaa6ecb8c15554509d0503a414b4d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:33:47 -0400 Subject: [PATCH 04/20] feat(mqtt): wire sparkplug.birthObservationWindowSeconds into the browse surface (R06:M-4) The authored window was persisted and consumed by nothing. It now (clamped 1-120 s) extends the Sparkplug OpenBudget, bounds a passive UntilStable-style settle on open (return after a 1 s quiet spell in NBIRTH/DBIRTH traffic), and bounds how long an operator rebirth request waits for the births it asked for before returning, so the re-rendered tree carries them. A caller deadline during the wait still reports the publish. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 4 +- .../MqttBrowseSession.cs | 80 +++++++++- .../MqttDriverBrowser.cs | 65 +++++++- .../MqttBirthObservationWindowTests.cs | 145 ++++++++++++++++++ 4 files changed, 284 insertions(+), 10 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index 8400ab42..d8620d65 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -117,7 +117,7 @@ driver are pure organisational grouping in the RawPath. "HostId": "", // Host Application identity — spBv1.0/STATE/{HostId} "ActAsPrimaryHost": false, // claim the primary host id (Primary-gated; see below) "RequestRebirthOnGap": true, // a seq gap asks the edge node to re-announce (NCMD) - "BirthObservationWindowSeconds": 15 // how long discovery collects births before calling it stable + "BirthObservationWindowSeconds": 15 // how long the browse picker collects births before calling them stable } } ``` @@ -128,7 +128,7 @@ driver are pure organisational grouping in the RawPath. | `HostId` | `""` | Sparkplug Host Application identity — the `spBv1.0/STATE/{HostId}` topic. Optional for observation; **required** to claim. Set without `ActAsPrimaryHost`, the driver only observes | | `ActAsPrimaryHost` | `false` | Claim the Sparkplug Primary Host Application identity. **Primary-gated** — only the node holding the redundancy Primary role claims it. See [Primary host](#primary-host-actasprimaryhost) | | `RequestRebirthOnGap` | `true` | On a detected sequence-number gap, publish a rebirth-request NCMD instead of running on stale metric state | -| `BirthObservationWindowSeconds` | `15` | Discovery's birth-collection window. Must be ≥ 1 | +| `BirthObservationWindowSeconds` | `15` | The Sparkplug browse picker's birth-collection window, clamped to 1–120 s (R06:M-4). The open budget grows by it; the open lingers while NBIRTH/DBIRTH traffic is still arriving (returning after a 1 s quiet spell — an idle plant costs 1 s); and an operator **Request rebirth** waits, bounded by it, for the births it asked for to arrive and go quiet for 500 ms, so the tree re-rendered afterwards carries them. A caller deadline during that wait still reports the publish. The runtime driver does not read it (its `DiscoverAsync` has no production caller — `SupportsOnlineDiscovery` is false) | `MqttDriverForm` authors all five in Sparkplug mode. The sub-object is **merged, never replaced** (a key a newer driver adds inside it survives an older AdminUI), and in `Plain` mode it is not touched at diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs index a231124b..9d4f6772 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs @@ -149,6 +149,9 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + /// The quiet spell that ends a rebirth request's wait once its births have started arriving. + private static readonly TimeSpan RebirthQuietInterval = TimeSpan.FromMilliseconds(500); + /// Bound on the courtesy DISCONNECT issued at dispose — teardown never hangs on a dead broker. private static readonly TimeSpan DisconnectTimeout = TimeSpan.FromSeconds(5); @@ -174,6 +177,7 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession private readonly ConcurrentQueue _publishedTopics = new(); private int _disposed; + private long _birthCount; private long _lastUsedTicksUtc = DateTime.UtcNow.Ticks; private int _nodeCount; private int _oversized; @@ -195,9 +199,16 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// The connected MQTT client; disconnected and disposed by . /// Optional logger; never receives credentials or payload bodies. /// Cap on recorded nodes; defaults to . - internal MqttBrowseSession(MqttMode mode, IMqttClient? client, ILogger? logger, int nodeCap = DefaultNodeCap) + /// + /// The Sparkplug birth-observation window: how long waits for the + /// births it asked for. (the test default) returns as soon as the NCMDs + /// are published. + /// + internal MqttBrowseSession( + MqttMode mode, IMqttClient? client, ILogger? logger, int nodeCap = DefaultNodeCap, TimeSpan birthWindow = default) { Mode = mode; + BirthWindow = birthWindow; _client = client; _logger = logger; _nodeCap = nodeCap; @@ -211,6 +222,56 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// internal int PublishCountForTest => Volatile.Read(ref _publishCount); + /// The Sparkplug birth-observation window this session was opened with (R06:M-4). + internal TimeSpan BirthWindow { get; } + + /// How many NBIRTH/DBIRTH messages this session has observed — the settle loop's signal. + internal long BirthCount => Interlocked.Read(ref _birthCount); + + /// + /// Waits until birth traffic has gone quiet for one interval, bounded by + /// — the UntilStable cadence of the birth-observation window. + /// + /// Reads the running birth count. + /// The count before the wait began. + /// + /// When true, quiet only counts once at least one birth has arrived past + /// — the shape a rebirth request needs, since its answer takes a broker round-trip to arrive. + /// + /// The quiet spell that ends the wait. + /// The hard ceiling; non-positive returns at once. + /// Cancels the wait. + /// A task that completes when the births have settled or the window has elapsed. + internal static async Task SettleBirthsAsync( + Func birthCount, + long baseline, + bool awaitFirstBirth, + TimeSpan quiet, + TimeSpan window, + CancellationToken cancellationToken) + { + if (window <= TimeSpan.Zero) return; + + var clock = System.Diagnostics.Stopwatch.StartNew(); + var last = baseline; + var sawBirth = !awaitFirstBirth; + while (clock.Elapsed < window) + { + var step = window - clock.Elapsed; + await Task.Delay(step < quiet ? step : quiet, cancellationToken).ConfigureAwait(false); + + var now = birthCount(); + if (now != last) + { + last = now; + sawBirth = true; + continue; + } + + if (sawBirth) return; + } + } + /// The topics this session published, in order — the discriminating view of the above. internal IReadOnlyList PublishedTopicsForTest => [.. _publishedTopics]; @@ -557,6 +618,8 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession if (!SparkplugTopic.TryParse(topic, out var parsed)) return; if (parsed.Type is not (SparkplugMessageType.NBIRTH or SparkplugMessageType.DBIRTH)) return; + Interlocked.Increment(ref _birthCount); + // The bound has to be on the whole message: unlike a plain payload, a birth must be parsed // entire to yield its metric list, and the parse runs on the shared dispatcher thread. if (payload.Length > MaxBirthPayloadBytes) @@ -898,6 +961,7 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession ArgumentException.ThrowIfNullOrWhiteSpace(scope); var targets = ResolveRebirthTargets(scope); + var baseline = BirthCount; foreach (var (groupId, edgeNodeId) in targets) { @@ -908,6 +972,20 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession .ConfigureAwait(false); } + // The birth-observation window (R06:M-4): wait — bounded — for the births just asked for to arrive + // and settle, so the tree the picker re-renders after this call already carries them. A courtesy, + // never a failure: the NCMDs are out, so a caller deadline here still reports the publish. + try + { + await SettleBirthsAsync( + () => BirthCount, baseline, awaitFirstBirth: true, RebirthQuietInterval, BirthWindow, cancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Published; the caller simply stopped waiting for the answer. + } + Touch(); return targets.Count; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs index 3ef6b659..8be8affc 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttDriverBrowser.cs @@ -39,6 +39,15 @@ public sealed class MqttDriverBrowser : IDriverBrowser /// Ceiling on the open budget — the picker must never hang on an unreachable broker. internal const int MaxOpenBudgetSeconds = 30; + /// Ceiling on the Sparkplug birth-observation window — an authored hour must not park a picker. + internal const int MaxBirthWindowSeconds = 120; + + /// + /// How long the open waits for a quiet spell in birth traffic before handing the session back. An + /// idle plant (births are never retained) costs exactly this, once. + /// + internal static readonly TimeSpan BirthQuietInterval = TimeSpan.FromSeconds(1); + /// Marks the transient browse identity in the broker's client-id/session logs. internal const string BrowseClientIdPrefix = "-browse-"; @@ -77,7 +86,8 @@ public sealed class MqttDriverBrowser : IDriverBrowser openCts.CancelAfter(OpenBudget(opts)); var client = new MqttClientFactory().CreateMqttClient(); - var session = new MqttBrowseSession(opts.Mode, client, _logger); + var birthWindow = opts.Mode == MqttMode.SparkplugB ? BirthWindow(opts) : TimeSpan.Zero; + var session = new MqttBrowseSession(opts.Mode, client, _logger, birthWindow: birthWindow); try { // Runs on MQTTnet's dispatcher: record and return, nothing else. @@ -100,6 +110,29 @@ public sealed class MqttDriverBrowser : IDriverBrowser .Build(); await client.SubscribeAsync(subscribeOptions, openCts.Token).ConfigureAwait(false); + if (opts.Mode == MqttMode.SparkplugB) + { + // The birth-observation window (R06:M-4): births already in flight — an edge node mid-rebirth, + // a primary-host flap — are collected until the traffic goes quiet, bounded by the window, so + // the first RootAsync is not a snapshot taken in the middle of a burst. Passive: nothing is + // published to provoke a birth. An idle plant costs one quiet interval. + try + { + await MqttBrowseSession.SettleBirthsAsync( + () => session.BirthCount, + baseline: session.BirthCount, + awaitFirstBirth: false, + BirthQuietInterval, + birthWindow, + openCts.Token) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + // The open budget ran out mid-settle: what has been observed is still a usable session. + } + } + _logger.LogInformation( "AdminUI MQTT browse session opened against {Host}:{Port} in {Mode} mode observing " + "'{Filter}' (read-only).", @@ -179,12 +212,30 @@ public sealed class MqttDriverBrowser : IDriverBrowser options with { CleanSession = true }; /// - /// Whole-open bound (connect + subscribe), clamped to - /// – around the config's - /// own connect deadline. + /// Whole-open bound: connect + subscribe, clamped to + /// – around the config's own + /// connect deadline — plus, in Sparkplug mode, the the open may spend + /// collecting births. /// /// The browse configuration. - /// The clamped open budget. - internal static TimeSpan OpenBudget(MqttDriverOptions options) => - TimeSpan.FromSeconds(Math.Clamp(options.ConnectTimeoutSeconds, MinOpenBudgetSeconds, MaxOpenBudgetSeconds)); + /// The open budget. + internal static TimeSpan OpenBudget(MqttDriverOptions options) + { + var connect = TimeSpan.FromSeconds( + Math.Clamp(options.ConnectTimeoutSeconds, MinOpenBudgetSeconds, MaxOpenBudgetSeconds)); + return options.Mode == MqttMode.SparkplugB ? connect + BirthWindow(options) : connect; + } + + /// + /// The authored sparkplug.birthObservationWindowSeconds, clamped to 1– + /// s: how long the browse surface collects NBIRTH/DBIRTH traffic before calling the observed metric set + /// stable (on open, and after an operator rebirth request). + /// + /// The browse configuration. + /// The clamped window. + internal static TimeSpan BirthWindow(MqttDriverOptions options) => + TimeSpan.FromSeconds(Math.Clamp( + options.Sparkplug?.BirthObservationWindowSeconds ?? new MqttSparkplugOptions().BirthObservationWindowSeconds, + 1, + MaxBirthWindowSeconds)); } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs new file mode 100644 index 00000000..b7f920af --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs @@ -0,0 +1,145 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser; +using SparkplugDataType = Org.Eclipse.Tahu.Protobuf.DataType; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; + +/// +/// R06:M-4. sparkplug.birthObservationWindowSeconds was authored, range-checked and +/// persisted by the AdminUI and consumed by nothing. It now bounds how long the Sparkplug browse surface +/// collects NBIRTH/DBIRTH traffic before calling the observed set stable: the open budget grows by it, +/// the open lingers while births are still arriving, and an operator rebirth request waits (bounded) for +/// the births it asked for, so the tree the picker re-renders already carries them. +/// +public sealed class MqttBirthObservationWindowTests +{ + private static MqttDriverOptions Sparkplug(int windowSeconds, int connectTimeoutSeconds = 10) => new() + { + Mode = MqttMode.SparkplugB, + ConnectTimeoutSeconds = connectTimeoutSeconds, + Sparkplug = new MqttSparkplugOptions { GroupId = "Plant1", BirthObservationWindowSeconds = windowSeconds }, + }; + + [Fact] + public void OpenBudget_Sparkplug_AddsTheBirthObservationWindow() + { + MqttDriverBrowser.OpenBudget(Sparkplug(windowSeconds: 15)).ShouldBe(TimeSpan.FromSeconds(10 + 15)); + MqttDriverBrowser.OpenBudget(Sparkplug(windowSeconds: 40)).ShouldBe(TimeSpan.FromSeconds(10 + 40)); + } + + [Fact] + public void OpenBudget_Plain_IgnoresTheSparkplugWindow() + { + var plain = Sparkplug(windowSeconds: 40) with { Mode = MqttMode.Plain }; + MqttDriverBrowser.OpenBudget(plain).ShouldBe(TimeSpan.FromSeconds(10)); + } + + [Theory] + [InlineData(0, 1)] + [InlineData(-5, 1)] + [InlineData(15, 15)] + [InlineData(100_000, 120)] + public void BirthWindow_IsClamped(int configured, int expectedSeconds) + => MqttDriverBrowser.BirthWindow(Sparkplug(configured)).ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + + /// An idle plant costs one quiet interval, not the whole window. + [Fact] + public async Task Settle_WithNoBirths_ReturnsAfterOneQuietInterval() + { + var reads = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => { reads++; return 0; }, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromMinutes(5), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + reads.ShouldBe(1); + } + + /// + /// While births keep arriving the settle keeps collecting; it returns on the first quiet interval + /// after the last one. Counted in reads, not wall-clock, so it cannot flake. + /// + [Fact] + public async Task Settle_KeepsCollectingWhileBirthsArrive_ThenReturnsOnQuiet() + { + var reads = 0; + long births = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => + { + reads++; + if (reads <= 5) births++; // five intervals with fresh births, then silence + return births; + }, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromMinutes(5), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + reads.ShouldBe(6); + } + + /// The window is a hard ceiling: a birth flood that never goes quiet still ends. + [Trait("Category", "SlowUnit")] + [Fact] + public async Task Settle_NeverQuiet_EndsAtTheWindow() + { + long births = 0; + await MqttBrowseSession.SettleBirthsAsync( + () => ++births, + baseline: 0, + awaitFirstBirth: false, + quiet: TimeSpan.FromMilliseconds(20), + window: TimeSpan.FromSeconds(1), + TestContext.Current.CancellationToken) + .WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + + births.ShouldBeGreaterThan(1); + } + + /// + /// A rebirth request waits for the birth it asked for. The fixed version cannot complete + /// synchronously (it awaits the settle), the unfixed one completes before the call returns — so the + /// IsCompleted check is deterministic, not a timing race. The birth then releases it. + /// + [Fact] + public async Task RequestRebirth_WaitsForTheRequestedBirths_ThenReturns() + { + await using var s = new MqttBrowseSession( + MqttMode.SparkplugB, client: null, logger: null, birthWindow: TimeSpan.FromMinutes(5)); + + var request = s.RequestRebirthAsync("Plant1/EdgeA", TestContext.Current.CancellationToken); + request.IsCompleted.ShouldBeFalse("the request must wait for the births it asked for"); + + s.ObserveBirthForTest("Plant1", "EdgeA", null, "Uptime", SparkplugDataType.Int64); + + (await request.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBe(1); + s.PublishCountForTest.ShouldBe(1); + } + + /// + /// The wait is a courtesy, never a failure: the NCMD was already published, so a caller deadline + /// during the settle returns the published count instead of reporting an error for a request that + /// succeeded. + /// + [Fact] + public async Task RequestRebirth_CallerCancelsDuringTheSettle_StillReportsThePublish() + { + await using var s = new MqttBrowseSession( + MqttMode.SparkplugB, client: null, logger: null, birthWindow: TimeSpan.FromMinutes(5)); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + + var request = s.RequestRebirthAsync("Plant1/EdgeA", cts.Token); + request.IsCompleted.ShouldBeFalse(); + await cts.CancelAsync(); + + (await request.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBe(1); + } +} From c14a3388f1d83b2e2c3fee52603b6533fa7e7e8d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:35:55 -0400 Subject: [PATCH 05/20] fix(mqtt): Sparkplug array and unsupported-type metrics are not bindable from the browser (R06:M-5) The ingestor refuses array metrics and unmapped types, yet DescribeMetric still stated their binding tuple, so a browse-commit produced a tag stuck at BadWaitingForInitialData forever. Those leaves now state no address (the commit refuses them) and say why in the descriptor line. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 5 ++++- .../MqttBrowseSession.cs | 13 +++++++++++-- .../MqttBrowseSessionTests.cs | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index d8620d65..b9e62582 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -249,7 +249,10 @@ One subscription (`spBv1.0/{GroupId}/#`) feeds a per-edge-node state machine [Structured metrics](#structured-metrics-template--dataset--propertyset). - **Unsupported datatypes are skipped with a warning, never guessed**: `PropertySetList`, `Unknown`, extension values, and any `Template`/`DataSet` that cannot be flattened honestly. `Int8`→`Int16` and - `UInt8`→`UInt16` widen; `*Array` variants map to the element type with the array bit set. + `UInt8`→`UInt16` widen; `*Array` variants map to the element type with the array bit set — but + **array metrics are not read in v1** (warned once, never fed), so the browse picker offers them, and any + metric whose type is unsupported, **without an address**: the browse-commit refuses them rather than + committing a tag that waits for initial data forever (R06:M-5). - **Ingest state is reset at every session/authoring seam** — a reconnect or a `ReinitializeAsync` must not carry a previous session's alias table into a new one. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs index 9d4f6772..b6bf8d9c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs @@ -441,16 +441,25 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession var mapped = metric.DataType?.ToDriverDataType(); var sparkplugName = metric.DataType?.ToString() ?? nameof(SparkplugDataType.Unknown); + var isArray = metric.DataType?.IsSparkplugArray() ?? false; + var descriptor = new StringBuilder(metric.BirthType.ToString()).Append(" · ").Append(sparkplugName); if (metric.Alias is { } alias) descriptor.Append(" · alias ").Append(alias); if (mapped is null) descriptor.Append(" · unsupported in v1 — not bindable"); + else if (isArray) descriptor.Append(" · array metrics are not read in v1 — not bindable"); + + // Unbindable = the ingestor refuses it: an unmapped type, or ANY array variant (SparkplugIngestor + // gates on the array bit and never feeds the tag). Such a leaf states NO address, so the + // browse-commit refuses it at commit instead of writing a tag that sits at + // BadWaitingForInitialData forever (R06:M-5). Labelling it alone was not enough — it still bound. + var bindable = mapped is not null && !isArray; return new AttributeInfo( metric.Name, mapped?.ToString() ?? sparkplugName, - metric.DataType?.IsSparkplugArray() ?? false, + isArray, descriptor.ToString(), - AddressFields: BuildMetricAddressFields(metric.Name, scope)); + AddressFields: bindable ? BuildMetricAddressFields(metric.Name, scope) : null); } /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs index bbc14bc1..fd624dde 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBrowseSessionTests.cs @@ -779,6 +779,13 @@ public sealed class MqttBrowseSessionTests "Plant1/EdgeA::Profile", TestContext.Current.CancellationToken)).ShouldHaveSingleItem(); a.DriverDataType.ShouldBe(nameof(Core.Abstractions.DriverDataType.Int32)); // element type a.IsArray.ShouldBeTrue(); + + // R06:M-5: the ingestor refuses array metrics ("not supported in v1"), so the browser must not + // offer one as bindable — no stated address means the browse-commit refuses it in words, instead + // of committing a tag that sits at BadWaitingForInitialData forever. The scalar control is + // SparkplugAttributes_StateTheBindingTuple_SoTheCommitNeverParsesTheNodeId. + a.AddressFields.ShouldBeNull(); + a.SecurityClass.ShouldContain("not bindable"); } [Fact] @@ -796,6 +803,7 @@ public sealed class MqttBrowseSessionTests Enum.TryParse(a.DriverDataType, ignoreCase: true, out _) .ShouldBeFalse(); // not a guessed type — unmappable, and visibly so a.SecurityClass.ShouldContain("unsupported"); + a.AddressFields.ShouldBeNull(); // R06:M-5 — labelled AND unbindable, not just labelled } [Fact] From 6a51025012522971f469ef4c08ffac4932aba94e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:38:29 -0400 Subject: [PATCH 06/20] perf(mqtt): wildcard topic trie and a shared strict UTF-8 decoder (R06:M-6, M-7) Wildcard-authored tags were matched by one MqttTopicFilterComparer call per tag per inbound message on the dispatcher thread. They now live in an allocation-free level trie whose candidates are confirmed by the same comparer (semantics unchanged; pinned differentially against the comparer). TryDecodeUtf8 reuses one static throw-on-invalid UTF8Encoding instead of allocating one per Raw/Scalar delivery. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../MqttSubscriptionManager.cs | 76 +++++++-- .../TopicFilterTrie.cs | 152 ++++++++++++++++++ .../TopicFilterTrieTests.cs | 96 +++++++++++ 3 files changed, 311 insertions(+), 13 deletions(-) create mode 100644 src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 1504328a..0c8b1a9a 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -48,7 +48,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt; /// accepted by the parser (warned at deploy, see MqttTagDefinitionFactory.Inspect), so /// matching uses MQTTnet's own rather than string /// equality — the same comparer a broker applies. Concrete topics stay a single dictionary -/// lookup; the linear wildcard scan runs only when a wildcard tag actually exists. +/// lookup; wildcard filters live in a level trie (TopicFilterTrie) whose candidates the comparer +/// confirms, so a thousand wildcard tags cost a walk over the topic's levels, not a thousand compares. /// /// /// Sparkplug seam. P1 ingests plain topics only. Sparkplug B decodes a @@ -74,6 +75,13 @@ public sealed class MqttSubscriptionManager /// public const int DefaultQos = 1; + /// + /// Throw-on-invalid UTF-8, shared (R06:M-7): it is stateless and thread-safe, and allocating one per + /// Raw/Scalar delivery was per-message garbage on the dispatcher thread. Same choice + /// MqttBrowseSession makes. + /// + private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + // OPC UA status codes (values verified against Opc.Ua.StatusCodes, not recalled). private const uint StatusGood = 0x00000000u; @@ -644,13 +652,14 @@ public sealed class MqttSubscriptionManager } // A published topic name may not itself contain a wildcard, so a wildcard-authored - // definition can never also be an exact-index hit: no tag is delivered twice. - foreach (var def in authored.Wildcards) + // definition can never also be an exact-index hit: no tag is delivered twice. The trie walks + // the topic's levels (R06:M-6 — this used to be one comparer call per wildcard tag, per + // message, on the shared dispatcher thread) and every candidate it yields is confirmed by + // MQTTnet's own comparer inside the visitor, so the semantics are the broker's. + if (authored.WildcardTrie.Count > 0) { - if (MqttTopicFilterComparer.Compare(topic, def.Topic) == MqttTopicFilterCompareResult.IsMatch) - { - Deliver(def, payload, retained, now, oversized, ref json); - } + var visitor = new WildcardDelivery(this, topic, payload, retained, now, oversized, ref json); + authored.WildcardTrie.Match(topic, ref visitor); } } finally @@ -1447,8 +1456,7 @@ public sealed class MqttSubscriptionManager { try { - text = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true) - .GetString(payload); + text = Utf8Strict.GetString(payload); return true; } catch (DecoderFallbackException) @@ -1476,6 +1484,47 @@ public sealed class MqttSubscriptionManager return $"mqtt:{_mode}:{id}:[{named}{overflow}]"; } + /// + /// Delivers each wildcard-trie candidate whose filter the MQTTnet comparer confirms. A ref struct + /// so the payload span and the fan-out's shared JSON parse ride along without a closure or a list. + /// + private ref struct WildcardDelivery : ITopicFilterVisitor + { + private readonly MqttSubscriptionManager _manager; + private readonly string _topic; + private readonly ReadOnlySpan _payload; + private readonly bool _retained; + private readonly DateTime _now; + private readonly bool _oversized; + private ref PayloadJson _json; + + public WildcardDelivery( + MqttSubscriptionManager manager, + string topic, + ReadOnlySpan payload, + bool retained, + DateTime now, + bool oversized, + ref PayloadJson json) + { + _manager = manager; + _topic = topic; + _payload = payload; + _retained = retained; + _now = now; + _oversized = oversized; + _json = ref json; + } + + public void Visit(MqttTagDefinition def) + { + if (MqttTopicFilterComparer.Compare(_topic, def.Topic) == MqttTopicFilterCompareResult.IsMatch) + { + _manager.Deliver(def, _payload, _retained, _now, _oversized, ref _json); + } + } + } + /// Opaque subscription identity; the diagnostic id names the mode and its topics. /// The human-readable identity surfaced for diagnostics. private sealed record MqttSubscriptionHandle(string DiagnosticId) : ISubscriptionHandle; @@ -1487,7 +1536,8 @@ public sealed class MqttSubscriptionManager /// /// Two topic indexes, and the difference is load-bearing. Delivery matches an /// incoming published topic, which never carries a wildcard, so it uses - /// (concrete only) plus a comparer scan of . + /// (concrete only) plus a walk whose candidates the + /// comparer confirms. /// Every path that instead reconstructs state from a subscribed filter string — /// reconnect re-subscribe and per-topic degradation — must use , which is /// keyed by the tag's authored topic including wildcard patterns, because that is what @@ -1501,12 +1551,12 @@ public sealed class MqttSubscriptionManager /// included. Keyed the same way _subscribedTopics and SUBACK outcomes are. /// /// Concrete topic → its definitions; the delivery fast path. - /// Definitions whose authored topic carries an MQTT wildcard. + /// Definitions whose authored topic carries an MQTT wildcard, by filter level. private sealed record AuthoredTable( FrozenDictionary ByRawPath, FrozenDictionary ByFilter, FrozenDictionary ByExactTopic, - MqttTagDefinition[] Wildcards) + TopicFilterTrie WildcardTrie) { public static readonly AuthoredTable Empty = Build(new Dictionary(StringComparer.Ordinal)); @@ -1556,7 +1606,7 @@ public sealed class MqttSubscriptionManager byRawPath.ToFrozenDictionary(StringComparer.Ordinal), byFilter.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.ToArray(), StringComparer.Ordinal), concrete.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.ToArray(), StringComparer.Ordinal), - [.. wildcards]); + TopicFilterTrie.Build(wildcards.Select(d => (d.Topic, d)))); } } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs new file mode 100644 index 00000000..976523f7 --- /dev/null +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/TopicFilterTrie.cs @@ -0,0 +1,152 @@ +using System.Collections.Frozen; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt; + +/// Receives each candidate a match produces. +/// The value type stored against each filter. +internal interface ITopicFilterVisitor +{ + /// Called once per value whose filter may match the topic. + /// The candidate. + void Visit(T value); +} + +/// +/// An immutable trie over MQTT topic FILTERS (+ / # wildcards), so matching one published +/// topic costs a walk over its levels instead of a comparer call per wildcard-authored tag (R06:M-6: +/// the linear scan ran on MQTTnet's shared dispatcher thread for every inbound message). +/// +/// +/// +/// A candidate generator, not the authority. The trie implements the plain MQTT 3.1.1 §4.7 +/// level rules — + is exactly one level (an empty one included), # is the parent level +/// and everything under it — and deliberately nothing subtler (the $-topic exclusion, shared +/// subscriptions). It can therefore only ever return a SUPERSET of the true matches, and every +/// caller confirms each candidate with MQTTnet's own MqttTopicFilterComparer, the comparer a +/// broker applies. Semantics stay byte-identical to the scan it replaces; only the work changes. +/// A differential test pins both halves against that comparer. +/// +/// +/// Allocation-free per match. Levels are walked as spans and looked up through the +/// dictionary's span alternate lookup; candidates are pushed to a caller-supplied visitor (a +/// ref struct in production) rather than collected into a list. +/// +/// +/// The value type stored against each filter. +internal sealed class TopicFilterTrie +{ + private readonly Node _root; + + private TopicFilterTrie(Node root, int count) + { + _root = root; + Count = count; + } + + /// An empty trie. + public static TopicFilterTrie Empty { get; } = Build([]); + + /// How many values were inserted. + public int Count { get; } + + /// Builds a trie from (filter, value) pairs. + /// The filters and their values. + /// The immutable trie. + public static TopicFilterTrie Build(IEnumerable<(string Filter, T Value)> entries) + { + var root = new Builder(); + var count = 0; + foreach (var (filter, value) in entries) + { + var node = root; + var levels = filter.Split('/'); + var terminated = false; + for (var i = 0; i < levels.Length; i++) + { + var level = levels[i]; + if (level == "#") + { + // '#' must be the last level; anything after it is a malformed filter the comparer + // will refuse anyway, so treating '#' as terminal here can only over-generate. + node.Hash.Add(value); + terminated = true; + break; + } + + node = level == "+" + ? node.Plus ??= new Builder() + : node.Children.TryGetValue(level, out var child) ? child : node.Children[level] = new Builder(); + } + + if (!terminated) node.Exact.Add(value); + count++; + } + + return new TopicFilterTrie(root.Freeze(), count); + } + + /// Pushes every value whose filter may match to . + /// The visitor; a ref struct is allowed. + /// A concrete published topic (never a filter). + /// Receives each candidate — a superset of the true matches; confirm each one. + public void Match(string topic, ref TVisitor visitor) + where TVisitor : ITopicFilterVisitor, allows ref struct + { + if (Count == 0 || topic is null) return; + Walk(_root, topic.AsSpan(), exhausted: false, ref visitor); + } + + private static void Walk(Node node, ReadOnlySpan rest, bool exhausted, ref TVisitor visitor) + where TVisitor : ITopicFilterVisitor, allows ref struct + { + // '#' here covers this level's parent-match AND every deeper level. + foreach (var value in node.Hash) visitor.Visit(value); + + if (exhausted) + { + foreach (var value in node.Exact) visitor.Visit(value); + return; + } + + var slash = rest.IndexOf('/'); + var level = slash < 0 ? rest : rest[..slash]; + var next = slash < 0 ? ReadOnlySpan.Empty : rest[(slash + 1)..]; + var nextExhausted = slash < 0; + + if (node.Children.Count > 0 + && node.Children.GetAlternateLookup>().TryGetValue(level, out var child)) + { + Walk(child, next, nextExhausted, ref visitor); + } + + if (node.Plus is { } plus) Walk(plus, next, nextExhausted, ref visitor); + } + + private sealed class Node(FrozenDictionary children, Node? plus, T[] exact, T[] hash) + { + public FrozenDictionary Children { get; } = children; + + public Node? Plus { get; } = plus; + + public T[] Exact { get; } = exact; + + public T[] Hash { get; } = hash; + } + + private sealed class Builder + { + public Dictionary Children { get; } = new(StringComparer.Ordinal); + + public Builder? Plus { get; set; } + + public List Exact { get; } = []; + + public List Hash { get; } = []; + + public Node Freeze() => new( + Children.ToFrozenDictionary(kv => kv.Key, kv => kv.Value.Freeze(), StringComparer.Ordinal), + Plus?.Freeze(), + [.. Exact], + [.. Hash]); + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs new file mode 100644 index 00000000..f21677e1 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs @@ -0,0 +1,96 @@ +using MQTTnet; +using Shouldly; +using Xunit; + +namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; + +/// +/// R06:M-6. The wildcard trie replaced a linear MqttTopicFilterComparer scan on the +/// dispatcher thread. Its contract is "a superset of the true matches, confirmed by the comparer", so it +/// is pinned differentially against that comparer — the oracle a broker uses — over a seeded random +/// space that deliberately includes empty levels, $-topics and wildcard-only filters. +/// +public sealed class TopicFilterTrieTests +{ + private static readonly string[] Levels = ["a", "b", "c", "", "$SYS", "x y"]; + + private sealed class Collect : ITopicFilterVisitor + { + public HashSet Seen { get; } = []; + + public void Visit(int value) => Seen.Add(value).ShouldBeTrue("a filter must be visited at most once per match"); + } + + [Fact] + public void Match_IsASupersetOfTheComparer_AndConfirmingYieldsExactlyTheComparersAnswer() + { + var random = new Random(744); + for (var round = 0; round < 200; round++) + { + var filters = Enumerable.Range(0, random.Next(1, 25)).Select(_ => RandomFilter(random)).ToList(); + var trie = TopicFilterTrie.Build(filters.Select((f, i) => (f, i))); + + for (var t = 0; t < 50; t++) + { + var topic = RandomTopic(random); + var expected = filters + .Select((f, i) => (f, i)) + .Where(p => MqttTopicFilterComparer.Compare(topic, p.f) == MqttTopicFilterCompareResult.IsMatch) + .Select(p => p.i) + .ToHashSet(); + + var collect = new Collect(); + trie.Match(topic, ref collect); + + collect.Seen.IsSupersetOf(expected).ShouldBeTrue($"topic '{topic}' missed a match; filters: {string.Join(" | ", filters)}"); + collect.Seen + .Where(i => MqttTopicFilterComparer.Compare(topic, filters[i]) == MqttTopicFilterCompareResult.IsMatch) + .ToHashSet() + .SetEquals(expected) + .ShouldBeTrue(); + } + } + } + + [Theory] + [InlineData("a/#", "a", true)] // '#' includes the parent level + [InlineData("a/#", "a/b/c", true)] + [InlineData("a/+", "a/", true)] // '+' matches an empty level + [InlineData("a/+", "a", false)] + [InlineData("+/+", "a/b", true)] + [InlineData("#", "anything/at/all", true)] + [InlineData("a/b", "a/b", true)] + [InlineData("a/b", "a/c", false)] + public void Match_FindsTheCanonicalCases(string filter, string topic, bool expected) + { + var trie = TopicFilterTrie.Build([(filter, 1)]); + var collect = new Collect(); + trie.Match(topic, ref collect); + collect.Seen.Contains(1).ShouldBe(expected); + } + + [Fact] + public void Empty_MatchesNothing() + { + var collect = new Collect(); + TopicFilterTrie.Empty.Match("a/b", ref collect); + collect.Seen.ShouldBeEmpty(); + } + + private static string RandomFilter(Random random) + { + var count = random.Next(1, 5); + var parts = new List(count); + for (var i = 0; i < count; i++) + { + var roll = random.Next(10); + if (roll < 2 && i == count - 1) { parts.Add("#"); break; } + parts.Add(roll < 5 ? "+" : Levels[random.Next(Levels.Length)]); + } + + return string.Join('/', parts); + } + + private static string RandomTopic(Random random) => + string.Join('/', Enumerable.Range(0, random.Next(1, 6)).Select(_ => Levels[random.Next(Levels.Length)])); +} From c529948745e0193ebcafc03c374c10b012ea9262 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:42:50 -0400 Subject: [PATCH 07/20] perf(mqtt): publish a write batch concurrently, serialised per topic; writes feed health (R06:M-8) WriteAsync published one awaited QoS 1 round-trip per request, so 50 writes to a stalled broker cost 50 publish deadlines. Different topics now publish concurrently (at most 8 in flight); same-topic writes stay in request order. A caller cancel still fills every slot. A successful publish advances LastSuccessfulRead. MiniBroker now frames packets properly and records/PUBACKs PUBLISHes. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 5 + .../MqttDriver.cs | 104 ++++++++++++--- .../MiniBroker.cs | 119 +++++++++++++++--- .../MqttDriverSessionLifecycleTests.cs | 73 +++++++++++ 4 files changed, 272 insertions(+), 29 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index b9e62582..d5c33835 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -491,6 +491,11 @@ with their real type; a birth type the encoder cannot represent (any `*Array`, D matters and is deliberately not configurable: a retained command would be replayed by the broker to the device on *every* future subscribe — including the device's own reconnect — so one stale write would be re-applied forever. +**A batch publishes concurrently, one topic at a time (R06:M-8).** Writes to *different* topics in one +`WriteAsync` go out together, at most 8 in flight, so a stalled broker costs a batch about one publish deadline +rather than one per request; writes to the *same* topic stay sequential in request order, so the last one wins. +A successful publish (PUBACK, or a sent QoS 0 command) advances `LastSuccessfulRead` like an inbound message. + **Plain QoS 1 is at-least-once.** Know the consequences before pointing a Plain `writeTopic` at a non-idempotent device topic: diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs index 26cdeab6..dec7569c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs @@ -104,6 +104,12 @@ public sealed class MqttDriver /// private const string DiscoveryFolderName = "Mqtt"; + /// + /// Ceiling on PUBLISHes one write batch keeps in flight (R06:M-8). Bounded so a large batch cannot + /// flood a broker's inflight window; different topics share it, one topic's writes stay sequential. + /// + internal const int MaxConcurrentPublishes = 8; + /// How often the host-connectivity poll samples . private static readonly TimeSpan HostProbeInterval = TimeSpan.FromSeconds(1); @@ -209,6 +215,9 @@ public sealed class MqttDriver private DriverHealth _health = new(DriverState.Unknown, null, null); + /// UTC ticks of the last successful write PUBLISH; 0 = none. Answer evidence for health. + private long _lastPublishUtcTicks; + /// Guards the generation bump against the generation-checked health write. private readonly object _healthLock = new(); @@ -446,7 +455,7 @@ public sealed class MqttDriver return stored; } - var lastMessage = connection.LastMessageUtc ?? stored.LastSuccessfulRead; + var lastMessage = Latest(connection.LastMessageUtc ?? stored.LastSuccessfulRead, LastPublishUtc); return connection.State switch { MqttConnectionState.Connected => new DriverHealth(DriverState.Healthy, lastMessage, stored.LastError), @@ -708,9 +717,12 @@ public sealed class MqttDriver { ArgumentNullException.ThrowIfNull(writes); - var results = new WriteResult[writes.Count]; + var results = new WriteResult?[writes.Count]; var connection = _connection; + // Pass 1 — everything that needs no broker: resolve, encode, refuse. What survives is grouped by + // destination topic in REQUEST order. + var byTopic = new Dictionary>(StringComparer.Ordinal); for (var i = 0; i < writes.Count; i++) { var w = writes[i]; @@ -738,22 +750,70 @@ public sealed class MqttDriver continue; } + if (!byTopic.TryGetValue(encoded.Topic, out var queue)) + { + byTopic[encoded.Topic] = queue = []; + } + + queue.Add((i, encoded, w.FullReference)); + } + + // Pass 2 — publish (R06:M-8). Different topics go out concurrently, at most MaxConcurrentPublishes + // in flight, so a stalled broker costs a batch ~one publish deadline instead of one per request. + // Writes to the SAME topic stay strictly sequential in request order: the device applies them in + // arrival order, and the last one must win. + if (byTopic.Count > 0) + { + using var slots = new SemaphoreSlim(MaxConcurrentPublishes, MaxConcurrentPublishes); + await Task.WhenAll(byTopic.Values.Select(queue => PublishQueueAsync(connection!, queue, slots, results, cancellationToken))) + .ConfigureAwait(false); + } + + // Every slot is filled — a caller cancellation leaves the unattempted ones BadCommunicationError + // rather than throwing, so the batch shape holds. + var filled = new WriteResult[writes.Count]; + for (var i = 0; i < filled.Length; i++) + { + filled[i] = results[i] ?? new WriteResult(StatusBadCommunicationError); + } + + return filled; + } + + /// Publishes one topic's writes in request order, each under a shared concurrency slot. + private async Task PublishQueueAsync( + MqttConnection connection, + List<(int Index, MqttEncodedWrite Encoded, string Reference)> queue, + SemaphoreSlim slots, + WriteResult?[] results, + CancellationToken cancellationToken) + { + foreach (var (index, encoded, reference) in queue) + { + try + { + await slots.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; // caller-cancelled: the remaining slots are filled by the caller + } + try { await connection .PublishAsync(encoded.Topic, encoded.Payload, encoded.Qos, retain: false, cancellationToken) .ConfigureAwait(false); - results[i] = new WriteResult(0u); + results[index] = new WriteResult(0u); + + // A PUBACK (or a sent QoS 0 command) is the broker answering — evidence the session works, + // so it feeds LastSuccessfulRead alongside inbound messages. + Interlocked.Exchange(ref _lastPublishUtcTicks, DateTime.UtcNow.Ticks); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - // Caller-cancelled: fill the remainder rather than throwing, so the batch shape holds. - for (var j = i; j < writes.Count; j++) - { - results[j] = new WriteResult(StatusBadCommunicationError); - } - - return results; + results[index] = new WriteResult(StatusBadCommunicationError); + return; } catch (Exception ex) { @@ -761,13 +821,15 @@ public sealed class MqttDriver ex, "MQTT driver '{DriverId}': write to '{Reference}' could not be published on '{Topic}'.", _driverInstanceId, - w.FullReference, + reference, encoded.Topic); - results[i] = new WriteResult(StatusBadCommunicationError); + results[index] = new WriteResult(StatusBadCommunicationError); + } + finally + { + slots.Release(); } } - - return results; } /// Maps an encode refusal to the status an OPC UA client sees. @@ -1528,6 +1590,20 @@ public sealed class MqttDriver OnHostStatusChanged?.Invoke(this, new HostStatusChangedEventArgs(HostName, previous, next)); } + /// The last successful write publish, or null. + private DateTime? LastPublishUtc + { + get + { + var ticks = Interlocked.Read(ref _lastPublishUtcTicks); + return ticks == 0 ? null : new DateTime(ticks, DateTimeKind.Utc); + } + } + + /// The later of two optional instants. + private static DateTime? Latest(DateTime? a, DateTime? b) + => a is null ? b : b is null ? a : (a > b ? a : b); + /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs index 77d0b762..5cb47e49 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MiniBroker.cs @@ -5,7 +5,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests; /// /// The smallest broker that can complete an MQTT 3.1.1 handshake: accept TCP, answer CONNECT with -/// a CONNACK, answer PINGREQ with PINGRESP, and nothing else. That is enough for MQTTnet to report +/// a CONNACK, answer PINGREQ with PINGRESP, SUBSCRIBE with a SUBACK (unless told not to), and record +/// PUBLISHes (PUBACKing QoS 1 only when told to). It never forwards a PUBLISH to a subscriber. That is enough for MQTTnet to report /// a real connection — which is what makes a real drop () drive a real /// DisconnectedAsync and a real reconnect, rather than a test seam standing in for one. /// @@ -107,6 +108,96 @@ internal sealed class MiniBroker : IDisposable return [.. ack]; } + /// + /// When true, a QoS 1 PUBLISH is answered with a PUBACK. False (the default) is a broker that takes + /// the PUBLISH and never acknowledges it — the stalled-broker shape a write deadline exists for. + /// + public bool AckPublish { get; set; } + + /// Every PUBLISH received, in arrival order, as (topic, payload). + public System.Collections.Concurrent.ConcurrentQueue<(string Topic, byte[] Payload)> Published { get; } = new(); + + /// Handles one complete packet; returns false when the connection must close. + private async Task HandlePacketAsync(NetworkStream stream, byte[] packet) + { + var packetType = packet[0] & 0xF0; + if (packetType == 0x10) + { + await stream.WriteAsync(new byte[] { 0x20, 0x02, 0x00, ConnAckReturnCode }, _cts.Token); + + // A real broker closes the socket straight after refusing, which is what drives + // MQTTnet's DisconnectedAsync. Mirroring it keeps the fake honest. + return ConnAckReturnCode == ReturnCodeAccepted; + } + + if (packetType == 0xC0) + { + await stream.WriteAsync(PingResp, _cts.Token); + } + else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(packet, packet.Length) is { } subAck) + { + await stream.WriteAsync(subAck, _cts.Token); + } + else if (packetType == 0x30) + { + var qos = (packet[0] >> 1) & 0x03; + var at = HeaderLength(packet); + var topicLength = (packet[at] << 8) | packet[at + 1]; + var topic = System.Text.Encoding.UTF8.GetString(packet, at + 2, topicLength); + at += 2 + topicLength; + byte idHigh = 0, idLow = 0; + if (qos > 0) + { + idHigh = packet[at]; + idLow = packet[at + 1]; + at += 2; + } + + Published.Enqueue((topic, packet[at..])); + if (qos == 1 && AckPublish) + { + await stream.WriteAsync(new byte[] { 0x40, 0x02, idHigh, idLow }, _cts.Token); + } + } + + return true; + } + + /// Length of a packet's fixed header (type byte + variable-length remaining length). + private static int HeaderLength(byte[] packet) + { + var at = 1; + while ((packet[at] & 0x80) != 0) at++; + return at + 1; + } + + /// Removes one complete packet from the front of , if one is there. + private static bool TryTakePacket(List pending, out byte[] packet) + { + packet = []; + if (pending.Count < 2) return false; + + var at = 1; + var remaining = 0; + var multiplier = 1; + byte digit; + do + { + if (at >= pending.Count) return false; + digit = pending[at++]; + remaining += (digit & 0x7F) * multiplier; + multiplier *= 128; + } + while ((digit & 0x80) != 0); + + var total = at + remaining; + if (pending.Count < total) return false; + + packet = pending.GetRange(0, total).ToArray(); + pending.RemoveRange(0, total); + return true; + } + /// Total TCP connections accepted since construction — counts reconnect attempts. public int AcceptedCount => Volatile.Read(ref _acceptedCount); @@ -178,7 +269,8 @@ internal sealed class MiniBroker : IDisposable private async Task ServeAsync(TcpClient client) { - var buffer = new byte[1024]; + var buffer = new byte[4096]; + var pending = new List(); try { var stream = client.GetStream(); @@ -195,25 +287,22 @@ internal sealed class MiniBroker : IDisposable continue; } - var packetType = buffer[0] & 0xF0; - if (packetType == 0x10) + // Frame properly: several packets can share one TCP read (concurrent publishes do), and + // one packet can span two. + pending.AddRange(buffer.AsSpan(0, read).ToArray()); + var close = false; + while (TryTakePacket(pending, out var packet)) { - await stream.WriteAsync(new byte[] { 0x20, 0x02, 0x00, ConnAckReturnCode }, _cts.Token); - - // A real broker closes the socket straight after refusing, which is what drives - // MQTTnet's DisconnectedAsync. Mirroring it keeps the fake honest. - if (ConnAckReturnCode != ReturnCodeAccepted) + if (!await HandlePacketAsync(stream, packet)) { + close = true; break; } } - else if (packetType == 0xC0) + + if (close) { - await stream.WriteAsync(PingResp, _cts.Token); - } - else if (packetType == 0x80 && AnswerSubscribe && TryBuildSubAck(buffer, read) is { } subAck) - { - await stream.WriteAsync(subAck, _cts.Token); + break; } } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 76d12cdd..6abfa293 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -288,6 +288,79 @@ public sealed class MqttDriverSessionLifecycleTests driver.GetHealth().State.ShouldBe(DriverState.Healthy); } + private static RawTagEntry WritableTag(string rawPath, string writeTopic) => new( + rawPath, + $$"""{"topic":"{{writeTopic}}/state","payloadFormat":"Scalar","dataType":"Float64","writeTopic":"{{writeTopic}}"}""", + WriteIdempotent: false); + + /// + /// R06:M-8. A write batch used to publish strictly serially, one awaited QoS 1 round-trip per + /// request — against a stalled broker, N writes × the publish deadline. Different topics now publish + /// concurrently: all four PUBLISHes reach a broker that never PUBACKs well inside ONE 30 s deadline + /// (serial code would have sent exactly one). Cancelling the batch then fills every slot, so the + /// batch shape holds. + /// + [Fact] + public async Task AWriteBatch_PublishesDifferentTopicsConcurrently() + { + using var broker = new MiniBroker(); // never PUBACKs + var options = BrokerOptions(broker.Port) with + { + ConnectTimeoutSeconds = 30, + RawTags = [.. Enumerable.Range(0, 4).Select(i => WritableTag($"Plant/Mqtt/dev1/Sp{i}", $"cmd/sp{i}"))], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var write = driver.WriteAsync( + [.. Enumerable.Range(0, 4).Select(i => new WriteRequest($"Plant/Mqtt/dev1/Sp{i}", (double)i))], + cts.Token); + + await WaitForAsync(() => broker.Published.Count == 4, "all four PUBLISHes to be in flight at once"); + write.IsCompleted.ShouldBeFalse("no PUBACK has arrived, so nothing can have completed"); + + await cts.CancelAsync(); + var results = await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + results.Count.ShouldBe(4); + results.ShouldAllBe(r => r.StatusCode == 0x80050000u); // BadCommunicationError, every slot filled + } + + /// + /// Writes to the SAME topic in one batch stay in request order — last write wins at the device — + /// while the batch as a whole is concurrent. The broker PUBACKs here, so every slot is Good — and + /// those acks advance LastSuccessfulRead (writes feed health). + /// + [Fact] + public async Task AWriteBatch_KeepsSameTopicWritesInRequestOrder() + { + using var broker = new MiniBroker { AckPublish = true }; + var options = BrokerOptions(broker.Port) with + { + RawTags = [WritableTag("Plant/Mqtt/dev1/Sp", "cmd/sp"), WritableTag("Plant/Mqtt/dev1/Other", "cmd/other")], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + driver.GetHealth().LastSuccessfulRead.ShouldBeNull("nothing has answered yet"); + + var results = await driver.WriteAsync( + [ + new WriteRequest("Plant/Mqtt/dev1/Sp", 1.0), + new WriteRequest("Plant/Mqtt/dev1/Other", 9.0), + new WriteRequest("Plant/Mqtt/dev1/Sp", 2.0), + new WriteRequest("Plant/Mqtt/dev1/Sp", 3.0), + ], + TestContext.Current.CancellationToken); + + results.ShouldAllBe(r => r.StatusCode == 0u); + broker.Published.Where(p => p.Topic == "cmd/sp") + .Select(p => System.Text.Encoding.UTF8.GetString(p.Payload)) + .ShouldBe(["1", "2", "3"]); + + // A PUBACK is the broker answering, so writes feed the evidence clock the comms-loss watchdog reads. + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); From 16f401b9e70f469a81463246bcfb47986881d218 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:46:21 -0400 Subject: [PATCH 08/20] fix(sql): dispose the poll reader's concurrency gate once no group holds it (R06:S-3) SqlPollReader's SemaphoreSlim was never disposed and a new reader is built per rebind. The reader is now IDisposable with a reference count (owner + one per in-flight group): the gate is disposed only when the last group - including a timed-out one still wedged in the provider - releases it, so nothing ever releases into a disposed semaphore. ApplyBinding disposes the replaced reader; the final DisposeAsync disposes the live one (Shutdown does not, since a re-init without a rebind keeps polling through it). Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 17 ++++- .../SqlPollReader.cs | 68 +++++++++++++++++-- .../SqlPollReaderTests.cs | 54 +++++++++++++++ .../SqlReinitConfigAdoptionTests.cs | 37 ++++++++++ 4 files changed, 171 insertions(+), 5 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 59ae4956..945fc32b 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -261,6 +261,10 @@ public sealed class SqlDriver _factory = _explicitFactory ?? dialect.Factory; _connectionString = connectionString; Endpoint = DescribeEndpoint(connectionString); + + // The reader being replaced owns a SemaphoreSlim (R06:S-3). Disposing it is safe even with a stale + // poll still inside it: the gate goes only when the last in-flight group releases it. + var previous = _reader; _reader = new SqlPollReader( _factory, connectionString, @@ -271,6 +275,7 @@ public sealed class SqlDriver nullIsBad: options.NullIsBad, resolve: rawPath => _resolver.TryResolve(rawPath, out var definition) ? definition : null, logger: _logger); + previous?.Dispose(); } /// True when the supplied DriverConfig JSON carries a real body. The bootstrapper always passes @@ -1150,7 +1155,17 @@ public sealed class SqlDriver /// await using does not leak the poll loops. /// /// A task that represents the asynchronous operation. - public async ValueTask DisposeAsync() => await TeardownAsync().ConfigureAwait(false); + public async ValueTask DisposeAsync() + { + await TeardownAsync().ConfigureAwait(false); + + // Only on the FINAL dispose: a Shutdown → Initialize cycle on the same instance keeps polling + // through this reader when the init does not rebind (no rebinder / placeholder config). + _reader.Dispose(); + } + + /// The live poll reader. Test observation of the S-3 lifecycle only. + internal SqlPollReader ReaderForTests => _reader; /// /// Shared teardown: stops every polled subscription and drops the authored table. Idempotent — safe diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index 04789c2d..da6ecfb5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -36,7 +36,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// no subscription state and no connection between polls, so the driver shell can wrap it in /// and own those concerns. /// -public sealed class SqlPollReader +public sealed class SqlPollReader : IDisposable { /// Minimum spacing between "your source violates the query contract" warnings, per reader. private static readonly TimeSpan ContractWarningInterval = TimeSpan.FromMinutes(1); @@ -52,12 +52,20 @@ public sealed class SqlPollReader /// /// Caps concurrently executing group queries — and therefore concurrently held connections - /// (design §8.4). Never disposed: a only needs disposal once its - /// AvailableWaitHandle has been touched, and not disposing it also removes the hazard of a - /// timed-out-but-still-running group releasing into a disposed semaphore. + /// (design §8.4). Disposed by only once every group that ever acquired it has + /// released it (), so a timed-out-but-still-running group can never release into a + /// disposed semaphore (R06:S-3 — it used to be never disposed, one per reinitialize). /// private readonly SemaphoreSlim _gate; + /// + /// Owner reference (1) plus one per in-flight group. drops the owner's; whoever + /// takes it to zero disposes . Zero is terminal — no group can start after it. + /// + private int _refs = 1; + + private int _disposed; + private long _lastContractWarningTicks; /// Initializes a new instance of the class. @@ -273,6 +281,22 @@ public sealed class SqlPollReader /// inside the deadline, so a poll behind a wedged group still returns on time — as BadTimeout. /// private async Task RunGroupAsync(SqlQueryPlan plan, CancellationToken cancellationToken) + { + // A reader replaced by a reinitialize (and disposed) can still be holding a stale poll: it runs no + // new group. Cancellation is the honest answer — the poll that asked has been superseded. + if (!TryAddRef()) throw new OperationCanceledException("The Sql poll reader was disposed."); + + try + { + return await RunGroupCoreAsync(plan, cancellationToken).ConfigureAwait(false); + } + finally + { + ReleaseRef(); + } + } + + private async Task RunGroupCoreAsync(SqlQueryPlan plan, CancellationToken cancellationToken) { var clock = Stopwatch.StartNew(); @@ -288,6 +312,9 @@ public sealed class SqlPollReader if (budget <= TimeSpan.Zero) return GroupResult.TimedOut; deadline.CancelAfter(budget); + // The work holds its OWN reference: it can outlive this method (a timed-out group is abandoned + // still running) and its finally releases the slot into _gate, which must therefore still exist. + AddRefForWork(); work = Task.Run( async () => { @@ -299,6 +326,7 @@ public sealed class SqlPollReader { _gate.Release(); deadline.Dispose(); + ReleaseRef(); } }, CancellationToken.None); @@ -334,6 +362,38 @@ public sealed class SqlPollReader } } + /// + /// Releases the concurrency gate once no group still holds it. Idempotent; safe while groups are in + /// flight (the last one out disposes it). A disposed reader starts no new group. + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) ReleaseRef(); + } + + /// True once has actually been disposed. Test observation only. + internal bool GateDisposedForTests => Volatile.Read(ref _refs) == 0; + + /// Takes a reference unless the reader is already fully released. + private bool TryAddRef() + { + while (true) + { + var current = Volatile.Read(ref _refs); + if (current == 0) return false; + if (Interlocked.CompareExchange(ref _refs, current + 1, current) == current) return true; + } + } + + /// A further reference for work started under an already-held one (so never from zero). + private void AddRefForWork() => Interlocked.Increment(ref _refs); + + /// Drops a reference; the one that reaches zero disposes the gate. + private void ReleaseRef() + { + if (Interlocked.Decrement(ref _refs) == 0) _gate.Dispose(); + } + /// How much of this group's operationTimeout budget is left; never negative. private TimeSpan Remaining(Stopwatch clock) { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index da658517..87a0d02a 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -522,6 +522,60 @@ public sealed class SqlPollReaderTests dialect.Factory, " ", dialect, CommandTimeout, OperationTimeout, 4, false, _ => null)); } + // ---- R06:S-3: the concurrency gate is released ---- + + /// An idle reader releases its gate on Dispose, and a disposed reader starts no new group. + [Fact] + public async Task Dispose_releasesTheGate_andADisposedReaderRunsNoNewGroup() + { + using var fixture = new SqlitePollFixture(); + var reader = Reader(fixture, KvTag("Speed", SqlitePollFixture.PresentKey)); + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + + reader.Dispose(); + reader.Dispose(); // idempotent + + reader.GateDisposedForTests.ShouldBeTrue(); + await Should.ThrowAsync(() => reader.ReadAsync(["Speed"], CancellationToken.None)); + } + + /// + /// Disposing while a timed-out group is still wedged inside the provider must NOT dispose the gate + /// under it — the zombie releases its slot into the gate when it finally finishes, which is exactly + /// the hazard that kept the gate undisposed before. The gate goes once the zombie is out. + /// + [Fact] + public async Task Dispose_withAWedgedGroupInFlight_waitsForItBeforeReleasingTheGate() + { + using var fixture = new SqlitePollFixture(); + var factory = new TrackingFactory(); + var reader = new SqlPollReader( + factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: TimeSpan.FromSeconds(30), operationTimeout: TimeSpan.FromMilliseconds(500), + maxConcurrentGroups: 1, nullIsBad: false, + resolve: Resolver(KvTag("Speed", SqlitePollFixture.PresentKey))); + + var locker = fixture.OpenNewConnection(); + Execute(locker, "BEGIN EXCLUSIVE"); + try + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + factory.Live.ShouldBe(1); // the zombie is still inside the provider — the control + + reader.Dispose(); + reader.GateDisposedForTests.ShouldBeFalse("a group still holds the gate"); + } + finally + { + Execute(locker, "ROLLBACK"); + locker.Dispose(); + } + + (await WaitUntilAsync(() => reader.GateDisposedForTests, TimeSpan.FromSeconds(30))) + .ShouldBeTrue("the last group out must release the gate"); + factory.Live.ShouldBe(0); + } + [Fact] public async Task ReadAsync_rejectsANullReferenceList() => await Should.ThrowAsync(async () => diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs index 1181de1a..f9223f94 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs @@ -101,6 +101,43 @@ public sealed class SqlReinitConfigAdoptionTests driver.Endpoint.ShouldContain("old-server"); } + /// + /// R06:S-3. Every rebind builds a new ; the replaced one (and its + /// SemaphoreSlim) used to be dropped undisposed, one per reinitialize. The control: the reader + /// that replaced it is live. + /// + [Fact] + public async Task A_rebind_disposes_the_reader_it_replaces() + { + using var _ = new ScopedEnvironmentVariable( + SqlConnectionStringResolver.EnvironmentVariablePrefix + RefA, + "Server=old-server;Database=OldDb;Integrated Security=true"); + using var __ = new ScopedEnvironmentVariable( + SqlConnectionStringResolver.EnvironmentVariablePrefix + RefB, + "Server=new-server;Database=NewDb;Integrated Security=true"); + + var driver = SqlDriverFactoryExtensions.CreateInstance( + "sql-1", $$"""{"provider":"SqlServer","connectionStringRef":"{{RefA}}"}""", loggerFactory: null); + var original = driver.ReaderForTests; + + try + { + await driver.ReinitializeAsync( + $$"""{"provider":"SqlServer","connectionStringRef":"{{RefB}}"}""", CancellationToken.None); + } + catch + { + // Liveness failure against a non-existent server is expected; the rebind ran first. + } + + driver.ReaderForTests.ShouldNotBeSameAs(original); + original.GateDisposedForTests.ShouldBeTrue(); + driver.ReaderForTests.GateDisposedForTests.ShouldBeFalse(); + + await driver.DisposeAsync(); + driver.ReaderForTests.GateDisposedForTests.ShouldBeTrue("the final dispose releases the live reader too"); + } + /// Sets an environment variable for the duration of a test and restores it after. The resolver /// reads process-global state, so a leaked variable would race every sibling test. private sealed class ScopedEnvironmentVariable : IDisposable From 91b7e9541b1d230907475396006328123af7790d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:47:18 -0400 Subject: [PATCH 09/20] fix(sql): withhold the JSON parser's text on a malformed driver config (R06:S-8) The factory quoted System.Text.Json's message (and attached the JsonException) into the failure that reaches InitializeFailed -> health LastError -> /hosts; a pasted connection-string literal is exactly the malformed text that lands there. The message now names the instance and the line/byte position only, as the probe and browser already do. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlDriverFactoryExtensions.cs | 8 ++++++- .../SqlDriverFactoryTests.cs | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs index a53e4390..3bb727e5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs @@ -194,8 +194,14 @@ public static class SqlDriverFactoryExtensions } catch (JsonException ex) { + // The parser's message AND the exception carrying it are withheld (R06:S-8), matching + // SqlDriverProbe and SqlDriverBrowser: System.Text.Json quotes the offending token, and a pasted + // connection-string literal is exactly the malformed text that lands here. This message becomes + // InitializeFailed -> health LastError -> /hosts. Line/byte position carry no content, so they stay. throw new InvalidOperationException( - $"Sql driver config for '{driverInstanceId}' is not valid JSON: {ex.Message}", ex); + $"Sql driver config for '{driverInstanceId}' is not valid JSON (line {ex.LineNumber + 1}, byte " + + $"{ex.BytePositionInLine}). The parser's message is withheld because the malformed text may " + + "carry a connection string."); } } diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs index 65d6a281..9d1265de 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverFactoryTests.cs @@ -25,6 +25,28 @@ public sealed class SqlDriverFactoryTests { // ---- the plan's headline legs ---- + /// + /// R06:S-8. A malformed config used to quote System.Text.Json's message — which echoes the + /// offending token — into the exception that becomes InitializeFailed → health + /// LastError → /hosts. A pasted connection string is exactly the malformed text that + /// reaches here, so the parser text (and the inner exception carrying it) is withheld, as the probe and + /// browser already do. The message still names the instance and says what is wrong. + /// + [Fact] + public void CreateInstance_malformedJson_withholdsTheParserText() + { + const string secret = "hunter2-do-not-log"; + var ex = Should.Throw(() => + SqlDriverFactoryExtensions.CreateInstance( + "d1", $$"""{"provider":"SqlServer","connectionStringRef": Password={{secret}} }""", null)); + + ex.Message.ShouldContain("d1"); + ex.Message.ShouldContain("not valid JSON"); + ex.Message.ShouldNotContain(secret); + ex.Message.ShouldNotContain("Password"); + ex.InnerException.ShouldBeNull(); + } + [Fact] public void CreateInstance_missingConnectionStringRef_throwsActionable() => Should.Throw(() => From 7f5d959fb938662509bd14dda1454ae0bf739382 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:49:24 -0400 Subject: [PATCH 10/20] feat(sql): surface IS_NULLABLE on browsed column attributes (R06:S-5) Every dialect projects IS_NULLABLE and nothing read it. The browse session now reads it (tolerantly: a dialect that does not project it reads as unknown) and the picker's descriptor line says 'nullable' / 'NOT NULL', so an operator knows a cell can come back NULL (Uncertain, or Bad under nullIsBad) before committing the tag. The runtime catalog loader keeps reading names only - nothing at runtime consumes nullability. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlBrowseSession.cs | 43 +++++++++++++++++-- .../SqlBrowseSessionTests.cs | 8 +++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs index f25aaff7..7feb6a0f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs @@ -48,6 +48,12 @@ internal sealed class SqlBrowseSession : IBrowseSession /// Column alias carrying the SQL type family name. private const string DataTypeColumn = "DATA_TYPE"; + /// + /// Column alias carrying YES/NO nullability. Every shipped dialect projects it (Oracle + /// normalises its Y/N); it used to be read by nothing (R06:S-5). + /// + private const string IsNullableColumn = "IS_NULLABLE"; + /// The TABLE_TYPE value marking a view rather than a base table. private const string ViewTableType = "VIEW"; @@ -158,7 +164,7 @@ internal sealed class SqlBrowseSession : IBrowseSession reference.Schema, reference.Table!, cancellationToken).ConfigureAwait(false); // Computed from the columns ALREADY READ for this call — the catalog is not queried a second time. - var timestampColumn = PreferredTimestampColumn(columns); + var timestampColumn = PreferredTimestampColumn([.. columns.Select(c => (c.Name, c.DataType))]); // Ordinal: a column NodeId is only ever minted from this same catalog output, so its case is the // catalog's own. A case-insensitive match would pick the wrong column on a case-sensitive collation @@ -191,7 +197,15 @@ internal sealed class SqlBrowseSession : IBrowseSession Name: column.Name, DriverDataType: _dialect.MapColumnType(column.DataType).ToString(), IsArray: false, - SecurityClass: ReadOnlySecurityClass, + // The picker's descriptor line ("{type} · {SecurityClass}") — display only. Nullability is + // what tells an operator a cell can come back NULL, which publishes Uncertain (or Bad under + // nullIsBad), before they commit the tag. + SecurityClass: column.IsNullable switch + { + true => ReadOnlySecurityClass + " · nullable", + false => ReadOnlySecurityClass + " · NOT NULL", + null => ReadOnlySecurityClass, + }, IsAlarm: false, AddressFields: addressFields), ]; @@ -356,7 +370,7 @@ internal sealed class SqlBrowseSession : IBrowseSession /// than an error: the catalog answering "no rows" is indistinguishable from a genuinely column-less /// relation, and neither is a reason to fail an operator's click. /// - private Task> ReadColumnsAsync( + private Task> ReadColumnsAsync( string schema, string table, CancellationToken ct) => QueryAsync( _dialect.ListColumnsSql, @@ -367,9 +381,30 @@ internal sealed class SqlBrowseSession : IBrowseSession }, static reader => ( Name: ReadString(reader, ColumnNameColumn), - DataType: ReadOptionalString(reader, DataTypeColumn) ?? string.Empty), + DataType: ReadOptionalString(reader, DataTypeColumn) ?? string.Empty, + IsNullable: ReadNullability(reader)), ct); + /// + /// Reads IS_NULLABLE as a tri-state: when a (custom) dialect does not + /// project it or the value is neither YES nor NO — "unknown" is said by omission, never + /// guessed. + /// + private static bool? ReadNullability(DbDataReader reader) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (!string.Equals(reader.GetName(i), IsNullableColumn, StringComparison.OrdinalIgnoreCase)) continue; + if (reader.IsDBNull(i)) return null; + var text = reader.GetValue(i)?.ToString()?.Trim(); + if (string.Equals(text, "YES", StringComparison.OrdinalIgnoreCase)) return true; + if (string.Equals(text, "NO", StringComparison.OrdinalIgnoreCase)) return false; + return null; + } + + return null; + } + /// /// Runs one catalog query under the gate and projects every row. The disposed check happens /// inside the gate wait so a dispose racing an in-flight call cannot be missed. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs index 6be390f9..0bd53ad5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/SqlBrowseSessionTests.cs @@ -78,7 +78,8 @@ public sealed class SqlBrowseSessionTests attribute.Name.ShouldBe(SqliteBrowseFixture.RealColumn); attribute.DriverDataType.ShouldBe(nameof(DriverDataType.Float64)); attribute.IsArray.ShouldBeFalse(); - attribute.SecurityClass.ShouldBe("ViewOnly"); + // R06:S-5: every dialect projects IS_NULLABLE and the picker now says so. The column is declared NULL. + attribute.SecurityClass.ShouldBe("ViewOnly · nullable"); attribute.IsAlarm.ShouldBeFalse(); } @@ -92,7 +93,10 @@ public sealed class SqlBrowseSessionTests SqliteDialect.MainSchema, SqliteBrowseFixture.KeyValueTable, SqliteBrowseFixture.TextColumn); var attributes = await session.AttributesAsync(nodeId, CancellationToken.None); - attributes.ShouldHaveSingleItem().DriverDataType.ShouldBe(nameof(DriverDataType.String)); + var attribute = attributes.ShouldHaveSingleItem(); + attribute.DriverDataType.ShouldBe(nameof(DriverDataType.String)); + // The NOT NULL control for the nullable case above — the descriptor tracks the catalog, not a constant. + attribute.SecurityClass.ShouldBe("ViewOnly · NOT NULL"); } /// From 0a5aa9fc43bed85fbc4638c83e787991fd1fc9bc Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 15:57:13 -0400 Subject: [PATCH 11/20] fix(sql): row-cap key-value and where-pair plans with a TOP/LIMIT N+1 dialect seam (R06:S-1) Key-value and where-pair plans had no row limit, so a selector pointed at a history or events table materialised every matching row on every poll. ISqlDialect gains RowLimitPrefix/RowLimitSuffix(n) (SQL Server TOP (n), Postgres LIMIT n, Oracle FETCH FIRST n ROWS ONLY); plans carry a RowCap (distinct keys, or 1) and ask for RowCap+1 rows; the reader also stops client-side at RowCap+1. Getting the extra row proves the selector ambiguous: the group publishes BadConfigurationError with a rate-limited warning. The reader reports those answered refusals so the driver counts them as answer evidence, and any refusal-only poll now restores Healthy after a comms fault. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Sql.md | 11 +++ .../ISqlDialect.cs | 25 +++++ .../OracleDialect.cs | 15 +++ .../PostgresDialect.cs | 14 +++ .../SqlServerDialect.cs | 16 ++++ .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 31 ++++--- .../SqlGroupPlanner.cs | 18 +++- .../SqlPollReader.cs | 93 +++++++++++++++---- .../SqlQueryPlan.cs | 10 +- .../OracleFlavouredSqliteDialect.cs | 4 + .../ColonMarkerDialect.cs | 4 + .../OracleDialectTests.cs | 9 ++ .../PostgresDialectTests.cs | 9 ++ .../SqlCatalogDefaultSchemaTests.cs | 4 + .../SqlCatalogGateDriverTests.cs | 4 + .../SqlDriverTests.cs | 37 +++++++- .../SqlGroupPlannerChunkingTests.cs | 6 +- .../SqlGroupPlannerTests.cs | 48 ++++++++-- .../SqlPollReaderTests.cs | 93 ++++++++++++++++++- .../SqlServerDialectTests.cs | 9 ++ .../SqliteDialect.cs | 14 +++ 21 files changed, 420 insertions(+), 54 deletions(-) diff --git a/docs/drivers/Sql.md b/docs/drivers/Sql.md index 7b7ad3ae..ffa68922 100644 --- a/docs/drivers/Sql.md +++ b/docs/drivers/Sql.md @@ -166,6 +166,17 @@ emits a `SqlQueryPlan` per distinct group key. Grouping is deterministic (groups order of their first member, members keep their input order, parameters keep first-appearance order), so an unchanged configuration produces byte-identical SQL. +**Every key-value and where-pair query is row-capped (R06:S-1).** A key-value plan asks for one row more +than it has distinct keys (`TOP (n+1)` / `LIMIT n+1` / `FETCH FIRST n+1 ROWS ONLY`), a where-pair wide-row plan +for two, and the reader stops reading there even if a provider ignores the limit. Receiving the extra row +proves the selector is ambiguous — a history/events table, or a where column that is not unique — so every +member of that group publishes `BadConfigurationError` with a rate-limited warning naming the table and +selector, instead of the old behaviour of materialising every matching row on every poll and publishing an +arbitrary "last row" at Good. A duplicate that fits inside the cap (a duplicated key alongside an absent one) +still resolves last-row-wins. `topByTimestamp` keeps its single-row limit. An over-cap answer, like every +other refusal (absent row, NULL cell, uncoercible cell), is the database answering: it advances +`LastSuccessfulRead` and restores `Healthy` after a comms fault. + ⚠️ **Group keys are case-sensitive on identifiers.** Two tags authored `dbo.T` and `DBO.T` plan as **two** groups — one extra round-trip, not a correctness problem, but worth knowing if a group count looks higher than expected. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs index c144ec62..e1069372 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/ISqlDialect.cs @@ -177,6 +177,31 @@ public interface ISqlDialect /// string SingleRowLimitSuffix { get; } + /// + /// The position's fragment for an -row limit — + /// T-SQL's "TOP (n) "; empty for a dialect that limits at the end. Same trailing-space rule. + /// + /// + /// Added for the key-value / where-pair row cap (R06:S-1): those plans used to carry no limit, so an + /// ambiguous selector against a history table materialised every matching row on every poll. The + /// planner asks for N + 1 rows — one more than the plan can use — so ambiguity stays observable + /// (the reader sees the extra row and publishes a configuration error) without unbounded reads. + /// + /// The row limit; at least 1. + /// The fragment, possibly empty. + /// is below 1. + string RowLimitPrefix(int rows); + + /// + /// The position's fragment for an -row limit — + /// " LIMIT n" / " FETCH FIRST n ROWS ONLY"; empty for a dialect that limits after + /// SELECT. Same leading-space rule. See . + /// + /// The row limit; at least 1. + /// The fragment, possibly empty. + /// is below 1. + string RowLimitSuffix(int rows); + /// Catalog query listing schemas — the browser's RootAsync level. Takes no parameters. string ListSchemasSql { get; } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs index 2dc72a83..1789959f 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/OracleDialect.cs @@ -124,6 +124,21 @@ public sealed class OracleDialect : ISqlDialect // statement shape, which this seam deliberately leaves with the planner. public string SingleRowLimitSuffix => " FETCH FIRST 1 ROWS ONLY"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + // The same 12c+ row-limiting clause as SingleRowLimitSuffix; valid after a WHERE with no ORDER BY. + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " FETCH FIRST " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture) + " ROWS ONLY"; + } + /// // Schemas visible to the connecting principal, from ALL_USERS (Oracle equates a schema with a // user), aliased to the TABLE_SCHEMA column name the catalog readers project by. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs index b332d912..6809b328 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/PostgresDialect.cs @@ -86,6 +86,20 @@ public sealed class PostgresDialect : ISqlDialect /// public string SingleRowLimitSuffix => " LIMIT 1"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " LIMIT " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + // The schemas holding relations this connection can actually see, system schemas excluded. // // Sourced from information_schema.tables, not information_schema.schemata - a deliberate diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs index 9db9848c..61057407 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Dialect/SqlServerDialect.cs @@ -87,6 +87,22 @@ public sealed class SqlServerDialect : ISqlDialect /// public string SingleRowLimitSuffix => string.Empty; + /// + // Parenthesised: TOP (n) is the documented modern form, and TOP n without parentheses is the + // backwards-compatibility spelling. + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return "TOP (" + rows.ToString(System.Globalization.CultureInfo.InvariantCulture) + ") "; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + /// public string ListSchemasSql => "SELECT DISTINCT TABLE_SCHEMA FROM INFORMATION_SCHEMA.TABLES ORDER BY TABLE_SCHEMA"; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 945fc32b..4a8936c9 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -519,9 +519,9 @@ public sealed class SqlDriver try { - var snapshots = await _reader.ReadAsync(fullReferences, cancellationToken).ConfigureAwait(false); - ObservePollOutcome(snapshots); - return snapshots; + var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); + ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups); + return result.Snapshots; } catch (OperationCanceledException) { @@ -608,7 +608,13 @@ public sealed class SqlDriver /// cadence rather than backed off from. /// /// The snapshots the reader returned for one poll. - internal void ObservePollOutcome(IReadOnlyList snapshots) + /// + /// Groups the database answered but the driver refused (an over-cap result set, R06:S-1). Their + /// members carry , which alone is not answer evidence + /// (the planner mints it before any I/O), so the count is what makes them count as the database + /// answering. + /// + internal void ObservePollOutcome(IReadOnlyList snapshots, int answeredRefusals = 0) { if (snapshots.Count == 0) return; @@ -625,6 +631,8 @@ public sealed class SqlDriver if (IsAnswerEvidence(snapshot.StatusCode)) answered++; } + answered += answeredRefusals; + if (unreachable > 0) { Degrade( @@ -660,17 +668,12 @@ public sealed class SqlDriver if (answered == 0) return; // authoring-class Bad only — not a statement about the database. - if (good == 0) - { - // The database answered, just not with a value: an absent row, a NULL cell, a cell that would - // not coerce. Those are data/authoring facts, so the health STATE is deliberately left exactly - // as it was — but they are still proof the transport works, and the evidence clock the - // comms-loss watchdog reads must say so. - TouchLastSuccessfulRead(); - TransitionHostTo(HostState.Running); - return; - } + // Good values, OR a database that answered with refusals only — an absent row, a NULL cell, an + // uncoercible cell, an over-cap result set. A refusal is the database answering: it advances the + // evidence clock AND restores Healthy after a comms fault, exactly as a Good read does. Leaving the + // state as it was latched a recovered database Degraded behind every poll that read only refusals. + // // Route through the Faulted guard, not a raw WriteHealth: a poll still in flight when a concurrent // ReinitializeAsync has recorded Faulted (the engine's ~5 s teardown wait is shorter than the 15 s // default operationTimeout, so a stale poll can still complete afterwards) must not flip Faulted back diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs index 9d1142b8..31c77351 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs @@ -230,15 +230,21 @@ public static class SqlGroupPlanner values.Add(keyValue); } + // Row cap (R06:S-1): one row per distinct key is all this plan can use, so it asks for ONE more — enough + // to see a source breaking the one-row-per-key contract, never enough to pull a history table's every + // matching row into memory on every poll. + var rowCap = values.Count; var sql = new StringBuilder("SELECT ") + .Append(dialect.RowLimitPrefix(rowCap + 1)) .Append(QuoteList(selected, dialect)) .Append(" FROM ").Append(QuoteQualifiedName(first.Table, dialect)) .Append(" WHERE ").Append(dialect.QuoteIdentifier(keyColumn)) .Append(" IN (").AppendJoin(", ", markers).Append(')') + .Append(dialect.RowLimitSuffix(rowCap + 1)) .ToString(); return new SqlQueryPlan( - SqlTagModel.KeyValue, groupKey, sql, markers, bindNames, values, members, selected); + SqlTagModel.KeyValue, groupKey, sql, markers, bindNames, values, members, selected, rowCap); } /// @@ -270,14 +276,18 @@ public static class SqlGroupPlanner // A where-pair wins over topByTimestamp; the parser already guarantees exactly one is populated. if (!Blank(first.RowSelectorColumn) && first.RowSelectorValue is not null) { + // Capped at TWO rows (R06:S-1): the selector must identify one row, and the second is the proof it + // does not. A TOP 1 alone would hide that ambiguity (see SqlPollReader.SelectWideRow); no cap at all + // materialised every matching row of an events table on every poll. var marker = dialect.ParameterMarker(RowSelectorBaseName); var sql = string.Concat( - "SELECT ", columns, " FROM ", table, - " WHERE ", dialect.QuoteIdentifier(first.RowSelectorColumn!), " = ", marker); + "SELECT ", dialect.RowLimitPrefix(2), columns, " FROM ", table, + " WHERE ", dialect.QuoteIdentifier(first.RowSelectorColumn!), " = ", marker, + dialect.RowLimitSuffix(2)); return new SqlQueryPlan( SqlTagModel.WideRow, groupKey, sql, [marker], [dialect.ParameterBindName(RowSelectorBaseName)], - [first.RowSelectorValue], members, selected); + [first.RowSelectorValue], members, selected, RowCap: 1); } if (!Blank(first.RowSelectorTopByTimestamp)) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index da6ecfb5..25e70a39 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -162,11 +162,24 @@ public sealed class SqlPollReader : IDisposable /// The database could not be reached (opening a connection failed). public async Task> ReadAsync( IReadOnlyList fullReferences, CancellationToken cancellationToken) + => (await ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false)).Snapshots; + + /// + /// plus what the snapshots alone cannot say: how many groups the database + /// ANSWERED with more rows than the plan can use (R06:S-1). Those groups publish + /// — the same code an un-plannable definition gets + /// before any I/O — so the driver needs this count to treat them as the answer evidence they are. + /// + /// The RawPaths to read. + /// The caller's token; see . + /// The snapshots and the answered-refusal count. + public async Task ReadDetailedAsync( + IReadOnlyList fullReferences, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(fullReferences); var results = new DataValueSnapshot[fullReferences.Count]; - if (results.Length == 0) return results; + if (results.Length == 0) return new SqlPollResult(results, 0); var readAt = DateTime.UtcNow; @@ -189,19 +202,23 @@ public sealed class SqlPollReader : IDisposable positions.Add(i); } - if (definitions.Count > 0) - await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false); + var rowCapExceeded = definitions.Count > 0 + ? await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false) + : 0; // Fail-safe. Nothing above should leave a hole, but "N in, N out" is a contract the caller indexes // blind — a null here would be a NullReferenceException in the publish fan-out, far from its cause. for (var i = 0; i < results.Length; i++) results[i] ??= new DataValueSnapshot(null, SqlStatusCodes.BadInternalError, null, readAt); - return results; + return new SqlPollResult(results, rowCapExceeded); } - /// Plans the resolved definitions, runs every group concurrently, and slices the results back. - private async Task ReadGroupsAsync( + /// + /// Plans the resolved definitions, runs every group concurrently, and slices the results back. + /// Returns how many groups came back over their row cap. + /// + private async Task ReadGroupsAsync( List definitions, Dictionary> slots, DataValueSnapshot[] results, @@ -227,7 +244,7 @@ public sealed class SqlPollReader : IDisposable foreach (var position in positions) results[position] = snapshot; } - return; + return 0; } // Groups run concurrently; _gate — not the task count — is what bounds open connections. Each group @@ -241,8 +258,14 @@ public sealed class SqlPollReader : IDisposable // leaves a sibling group running against a connection nobody is watching. var outcomes = await Task.WhenAll(groups).ConfigureAwait(false); + var rowCapExceeded = 0; for (var g = 0; g < plans.Count; g++) + { Slice(plans[g], outcomes[g], slots, results, readAt); + if (ReferenceEquals(outcomes[g], GroupResult.RowCapExceeded)) rowCapExceeded++; + } + + return rowCapExceeded; } /// @@ -502,8 +525,11 @@ public sealed class SqlPollReader : IDisposable columnIndex[selected[c]] = c; } + // The client-side half of the row cap (R06:S-1): the statement already asks for RowCap + 1 rows, but a + // provider that cannot spell a limit must not turn that into an unbounded read either. + var limit = plan.RowCap is { } cap ? cap + 1 : int.MaxValue; var rows = new List(); - while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + while (rows.Count < limit && await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) { var row = new object?[selected.Count]; for (var c = 0; c < selected.Count; c++) @@ -511,6 +537,25 @@ public sealed class SqlPollReader : IDisposable rows.Add(row); } + if (plan.RowCap is { } rowCap && rows.Count > rowCap) + { + if (ShouldWarnAboutContract()) + { + var first = plan.Members[0]; + var selector = plan.Model == SqlTagModel.KeyValue + ? string.Concat(first.KeyColumn, " IN (", rowCap.ToString(CultureInfo.InvariantCulture), " key(s))") + : string.Concat(first.RowSelectorColumn, " = ", first.RowSelectorValue); + _logger.LogWarning( + "Sql poll source '{Table}' returned more rows than the {Model} selector '{Selector}' can use " + + "(at most {Cap}) in one poll, so the selector is ambiguous — a history/events table, or a " + + "where column that is not unique. Its {Members} tag(s) publish BadConfigurationError and " + + "nothing past row {Limit} is read. Point the tag at a current-values table or a latest-row view.", + first.Table, plan.Model, selector, rowCap, plan.Members.Count, rowCap + 1); + } + + return GroupResult.RowCapExceeded; + } + return GroupResult.Ok(rows, columnIndex, typeNames); } @@ -609,15 +654,14 @@ public sealed class SqlPollReader : IDisposable /// emits no ORDER BY, so across polls the "last" row is whatever the server returned last, /// i.e. storage order. That is why more than one row is a reported contract violation and not merely /// a tie to break — the model's premise (design §5.3) is that the selector identifies one row. - /// Why the where-pair form is deliberately NOT given the dialect's single-row limit. A - /// TOP 1 with no ORDER BY is not deterministic either — it returns whichever row the - /// plan happens to produce first, so it would trade one arbitrary row for a differently arbitrary - /// one. Worse, it would destroy the only evidence this method has that the selector is ambiguous - /// (Rows.Count > 1), converting a loud misconfiguration into a permanently silent one — - /// precisely the failure mode this class's summary calls its top risk. The topByTimestamp - /// form keeps its row limit because there the ORDER BY at least fixes which rows can - /// be first. The accepted cost is that an ambiguous selector materialises every matching row for one - /// poll; the warning is what gets it fixed. + /// Why the where-pair form is capped at TWO rows, not one (R06:S-1). A TOP 1 with no + /// ORDER BY is not deterministic — it returns whichever row the plan produces first — and it would + /// destroy the only evidence that the selector is ambiguous, converting a loud misconfiguration into a + /// permanently silent one. No limit at all materialised every matching row of an events table on every + /// poll. Two rows keeps the evidence and bounds the read: a second row makes the group publish + /// BadConfigurationError in MaterialiseAsync, so a where-pair group never reaches this + /// method with more than one row. The multi-row warning below is now reached only by a plan with no + /// cap. /// ⚠ The topByTimestamp answer is only defined when the ordering column is unique /// ORDER BY ts DESC plus a one-row limit over a column with ties returns an /// arbitrary row among the tied ones, and — unlike the where-pair form — this method can never @@ -807,6 +851,13 @@ public sealed class SqlPollReader : IDisposable public static GroupResult Failed { get; } = new(SqlStatusCodes.BadCommunicationError, NoRows, NoColumns, []); + /// + /// The database answered with more rows than the plan can use — an ambiguous selector (R06:S-1). + /// A configuration fact, not a database fault; the driver counts it as answer evidence. + /// + public static GroupResult RowCapExceeded { get; } = + new(SqlStatusCodes.BadConfigurationError, NoRows, NoColumns, []); + /// The status every member inherits when is false. public uint FailureStatus { get; } @@ -835,6 +886,14 @@ public sealed class SqlPollReader : IDisposable } } +/// One poll's snapshots plus the answer evidence they cannot carry themselves. +/// One snapshot per requested reference, at the same index. +/// +/// Groups the database answered with more rows than their plan can use; their members carry +/// . +/// +public sealed record SqlPollResult(IReadOnlyList Snapshots, int RowCapExceededGroups); + /// /// The OPC UA status codes the Sql driver publishes, and the quality-class predicates its tests read /// them back with. diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs index 8e20c989..333a762c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs @@ -55,6 +55,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// The bare (unquoted) column names in 's SELECT list, in order and distinct — the /// reader's map from result-set ordinal to source column. /// +/// +/// The most rows this plan can USE — its distinct key count (key-value) or 1 (a where-pair wide row); +/// when the statement already limits itself (topByTimestamp). +/// asks for RowCap + 1, and the reader stops there: receiving the extra row proves the selector is +/// ambiguous (a history table, a non-unique where column) and the group publishes +/// BadConfigurationError instead of materialising every matching row on every poll (R06:S-1). +/// public sealed record SqlQueryPlan( SqlTagModel Model, object GroupKey, @@ -63,7 +70,8 @@ public sealed record SqlQueryPlan( IReadOnlyList ParameterBindNames, IReadOnlyList Parameters, IReadOnlyList Members, - IReadOnlyList SelectedColumns) + IReadOnlyList SelectedColumns, + int? RowCap = null) { // A plan is documented as safe to cache and reuse across polls, so it must not stay aliased to the // planner's mutable working lists — an IReadOnlyList holding a List is downcastable, and a diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs index a6868aa3..0987f4b8 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser.Tests/OracleFlavouredSqliteDialect.cs @@ -38,6 +38,10 @@ public sealed class OracleFlavouredSqliteDialect : ISqlDialect /// public string SingleRowLimitSuffix => _inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => _inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => _inner.RowLimitSuffix(rows); + /// public string ListSchemasSql => _inner.ListSchemasSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs index f514b1be..939a267b 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/ColonMarkerDialect.cs @@ -77,6 +77,10 @@ public sealed class ColonMarkerDialect : ISqlDialect /// public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + /// public string ListSchemasSql => Colonize(Inner.ListSchemasSql); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs index a0050cf7..70629ae4 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/OracleDialectTests.cs @@ -210,6 +210,15 @@ public class OracleDialectTests public void LivenessSql_selectsFromDual() => new OracleDialect().LivenessSql.ShouldBe("SELECT 1 FROM DUAL"); + [Fact] + public void RowLimit_isFetchFirstN_inTheSuffixPosition() + { + var d = new OracleDialect(); + d.RowLimitPrefix(3).ShouldBe(""); + d.RowLimitSuffix(3).ShouldBe(" FETCH FIRST 3 ROWS ONLY"); + Should.Throw(() => d.RowLimitSuffix(0)); + } + [Fact] public void SingleRowLimit_isFetchFirst_inTheSuffixPosition_withItsOwnLeadingSpace() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs index 2f56b7d1..de681931 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/PostgresDialectTests.cs @@ -226,6 +226,15 @@ public class PostgresDialectTests d.ListColumnsSql.ShouldContain("\"IS_NULLABLE\""); } + [Fact] + public void RowLimit_isLimitN_inTheSuffixPosition() + { + var d = new PostgresDialect(); + d.RowLimitPrefix(3).ShouldBe(""); + d.RowLimitSuffix(3).ShouldBe(" LIMIT 3"); + Should.Throw(() => d.RowLimitSuffix(0)); + } + [Fact] public void SingleRowLimit_isLimit1_inTheSuffixPosition_withItsOwnLeadingSpace() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs index 0c72ca4a..2f8a1f47 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogDefaultSchemaTests.cs @@ -113,6 +113,10 @@ public sealed class SqlCatalogDefaultSchemaTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql => twoSchemas ? Inner.ListSchemasSql + " UNION ALL SELECT 'other' AS TABLE_SCHEMA" : Inner.ListSchemasSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs index f0e5d373..017c0bd9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlCatalogGateDriverTests.cs @@ -374,6 +374,10 @@ public sealed class SqlCatalogGateDriverTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql { get; } = listSchemasSql; public string DefaultSchemaSql => Inner.DefaultSchemaSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 492cc05d..6dd21ce9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -272,14 +272,15 @@ public sealed class SqlDriverTests // ---- the comms-loss watchdog's evidence clock (Driver.Sql-006) ---- [Fact] - public async Task ReadAsync_whenTheDatabaseAnswersWithNoRow_advancesTheEvidenceClock_withoutChangingHealth() + public async Task ReadAsync_whenTheDatabaseAnswersWithNoRow_advancesTheEvidenceClock_andRestoresHealthy() { // Driver.Sql-006. LastSuccessfulRead is what DriverInstanceActor's comms-loss watchdog reads: a // non-Healthy driver whose clock has been frozen for 90 s is force-reconnected (Gitea #527), on the // premise that the clock is the honest evidence the transport works. BadNoData IS that evidence — // the query ran and the row is absent — so a driver whose key set was deleted must not be - // reconnected every 90 s against a database answering every poll on time. The health STATE stays - // exactly as it was: an absent row is a data fact, not a connectivity one. + // reconnected every 90 s against a database answering every poll on time. And since the database + // answered, the driver is Healthy again (lane rule: a refusal means the device answered — it used to + // leave the Degraded latched behind every refusal-only poll). using var fixture = new SqlitePollFixture(); await using var driver = NewDriver(fixture, KvEntry("Missing", SqlitePollFixture.AbsentKey)); await driver.InitializeAsync(ConfigJson, CancellationToken.None); @@ -295,7 +296,7 @@ public sealed class SqlDriverTests snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); var health = driver.GetHealth(); health.LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); // RED pre-fix: frozen - health.State.ShouldBe(DriverState.Degraded); // classification untouched + health.State.ShouldBe(DriverState.Healthy); // the database answered } [Fact] @@ -356,6 +357,34 @@ public sealed class SqlDriverTests driver.GetHealth().LastSuccessfulRead.ShouldBe(before); } + /// + /// R06:S-1. An over-cap result set publishes BadConfigurationError — the code the planner also + /// mints before any I/O, which on its own is NOT answer evidence (theory above). The reader's + /// answered-refusal count is what makes it count: the database answered, so the clock advances and a + /// comms-Degraded driver is Healthy again. Control: the same snapshot with no count leaves both alone. + /// + [Fact] + public async Task ObservePollOutcome_aRowCapRefusal_isAnswerEvidence_andRestoresHealthy() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + var refused = new DataValueSnapshot(null, SqlStatusCodes.BadConfigurationError, null, DateTime.UtcNow); + + driver.ObservePollOutcome([new DataValueSnapshot(null, SqlStatusCodes.BadTimeout, null, DateTime.UtcNow)]); + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + var before = driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); + await Task.Delay(20, TestContext.Current.CancellationToken); + + driver.ObservePollOutcome([refused]); // no count: planner-minted, no evidence + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + driver.GetHealth().LastSuccessfulRead.ShouldBe(before); + + driver.ObservePollOutcome([refused], answeredRefusals: 1); // the database answered, over-cap + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs index d8bc6c8a..dd6920c2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerChunkingTests.cs @@ -150,7 +150,7 @@ public sealed class SqlGroupPlannerChunkingTests plan.ParameterNames.ShouldBe( Enumerable.Range(0, plan.Parameters.Count) .Select(i => "@k" + i.ToString(CultureInfo.InvariantCulture))); - plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); + plan.SqlText.ShouldContain("IN (" + string.Join(", ", plan.ParameterNames) + ")"); } } @@ -297,6 +297,10 @@ public sealed class SqlGroupPlannerChunkingTests public string SingleRowLimitSuffix => Inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => Inner.RowLimitPrefix(rows); + + public string RowLimitSuffix(int rows) => Inner.RowLimitSuffix(rows); + public string ListSchemasSql => Inner.ListSchemasSql; public string DefaultSchemaSql => Inner.DefaultSchemaSql; diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs index 344f4447..058ea516 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs @@ -29,6 +29,38 @@ public class SqlGroupPlannerTests RowSelectorColumn: selectorColumn, RowSelectorValue: selectorValue, RowSelectorTopByTimestamp: topByTimestamp); + // ---- R06:S-1: the row cap ---- + + /// + /// A key-value plan asks for one row more than it has distinct keys, and a where-pair wide-row plan for + /// two: enough to SEE an ambiguous selector, never enough to materialise a history table every poll. + /// The topByTimestamp form keeps its single-row limit and needs no cap. + /// + [Fact] + public void Plans_carryARowCap_andAskForOneRowMore() + { + var plans = SqlGroupPlanner.Plan( + [ + KvTag("A", "dbo.TagValues", "tag_name", "k0", "num_value", null), + KvTag("B", "dbo.TagValues", "tag_name", "k1", "num_value", null), + KvTag("C", "dbo.TagValues", "tag_name", "k1", "num_value", null), // same key: one row serves both + WideTag("W", "dbo.LatestStatus", "oven_temp", selectorColumn: "station_id", selectorValue: "7"), + WideTag("T", "dbo.Readings", "oven_temp", topByTimestamp: "sample_ts"), + ], new SqlServerDialect()).ToList(); + + var kv = plans.Single(p => p.Model == SqlTagModel.KeyValue); + kv.RowCap.ShouldBe(2); + kv.SqlText.ShouldStartWith("SELECT TOP (3) "); + + var wherePair = plans.Single(p => p.Parameters.Count == 1 && p.Model == SqlTagModel.WideRow); + wherePair.RowCap.ShouldBe(1); + wherePair.SqlText.ShouldStartWith("SELECT TOP (2) "); + + var top = plans.Single(p => p.Parameters.Count == 0); + top.RowCap.ShouldBeNull(); + top.SqlText.ShouldStartWith("SELECT TOP 1 "); + } + // ---- the plan's golden case ---- [Fact] @@ -58,7 +90,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].ParameterNames.ShouldBe(new[] { "@k0" }); plans[0].SelectedColumns.ShouldBe(new[] { "tag_name", "num_value", "sample_ts" }); plans[0].Model.ShouldBe(SqlTagModel.KeyValue); @@ -75,7 +107,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].TimestampColumn.ShouldBeNull(); } @@ -181,7 +213,7 @@ public class SqlGroupPlannerTests var plan = plans[0]; plan.ParameterNames.Count.ShouldBe(plan.Parameters.Count); - plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); + plan.SqlText.ShouldEndWith("IN (" + string.Join(", ", plan.ParameterNames) + ")"); // T-SQL caps at the front plan.Parameters.ShouldBe(new object[] { "k0", "k1", "k2" }); } @@ -201,7 +233,7 @@ public class SqlGroupPlannerTests KvTag("B", "TagValues", "tag_name", "Line1.Temp", "num_value", null), ], new ColonMarkerDialect()).ToList(); - plans[0].SqlText.ShouldEndWith("IN (:k0, :k1)"); + plans[0].SqlText.ShouldEndWith("IN (:k0, :k1) LIMIT 3"); plans[0].SqlText.ShouldNotContain("@"); plans[0].ParameterNames.ShouldBe(new[] { ":k0", ":k1" }); } @@ -213,7 +245,7 @@ public class SqlGroupPlannerTests [WideTag("A", "LatestStatus", "oven_temp", selectorColumn: "station_id", selectorValue: "7")], new ColonMarkerDialect()).ToList(); - plans[0].SqlText.ShouldEndWith("= :w"); + plans[0].SqlText.ShouldEndWith("= :w LIMIT 2"); plans[0].SqlText.ShouldNotContain("@"); plans[0].ParameterNames.ShouldBe(new[] { ":w" }); } @@ -277,7 +309,7 @@ public class SqlGroupPlannerTests plans.Count.ShouldBe(1); plans[0].SqlText.ShouldBe( - "SELECT [oven_temp], [pressure] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); + "SELECT TOP (2) [oven_temp], [pressure] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); plans[0].ParameterNames.ShouldBe(new[] { "@w" }); plans[0].Parameters.ShouldBe(new object[] { "7" }); // the whereValue is bound, never inlined plans[0].SelectedColumns.ShouldBe(new[] { "oven_temp", "pressure" }); @@ -313,7 +345,7 @@ public class SqlGroupPlannerTests plans.Count.ShouldBe(1); plans[0].SelectedColumns.ShouldBe(new[] { "oven_temp", "pressure", "sample_ts" }); plans[0].SqlText.ShouldBe( - "SELECT [oven_temp], [pressure], [sample_ts] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); + "SELECT TOP (2) [oven_temp], [pressure], [sample_ts] FROM [dbo].[LatestStatus] WHERE [station_id] = @w"); } [Fact] @@ -424,7 +456,7 @@ public class SqlGroupPlannerTests // A column name is an identifier — it cannot be parameterized, so the guarantee is that it is // bracket-quoted with ] doubled, leaving one (nonexistent) identifier rather than executable SQL. plans[0].SqlText.ShouldBe( - "SELECT [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (2) [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); } // ---- contract edges ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index 87a0d02a..4181449c 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -187,7 +187,7 @@ public sealed class SqlPollReaderTests } [Fact] - public async Task ReadAsync_wideRow_whenTheSelectorMatchesManyRows_warnsInsteadOfSilentlyPickingOne() + public async Task ReadAsync_wideRow_whenTheSelectorMatchesManyRows_publishesAConfigurationError_andWarns() { using var fixture = new SqlitePollFixture(); // A second row for the SAME station. The where-pair form emits no ORDER BY and no row limit, so @@ -201,8 +201,11 @@ public sealed class SqlPollReaderTests var snapshots = await reader.ReadAsync(["Oven"], CancellationToken.None); - // Still a value — the reader degrades to "last row wins", exactly as the key-value model does. - snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + // R06:S-1: the where-pair plan now asks for TWO rows (one more than it can use). Getting the second is + // proof the selector is ambiguous, so the group publishes a configuration error rather than a + // storage-order-dependent "last row" value — and nothing past that second row is ever read. + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[0].Value.ShouldBeNull(); var warning = logger.Entries.ShouldHaveSingleItem(); warning.Level.ShouldBe(LogLevel.Warning); @@ -277,13 +280,93 @@ public sealed class SqlPollReaderTests $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + $"VALUES ('{SqlitePollFixture.PresentKey}', 99.0, '2026-07-24T10:00:09Z')"); - var reader = Reader(fixture, KvTag("Speed", SqlitePollFixture.PresentKey)); + // Two keys, one absent: the duplicate fits inside the row cap (2), so it is not provably a history + // table — the last-row-wins rule still applies there (the cap test below covers the other side). + var reader = Reader(fixture, + KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Gone", SqlitePollFixture.AbsentKey)); - var snapshots = await reader.ReadAsync(["Speed"], CancellationToken.None); + var snapshots = await reader.ReadAsync(["Speed", "Gone"], CancellationToken.None); // Design §3.6: the contract is one row per key; when a source breaks it the reader is at least // deterministic — last occurrence in reader order — rather than silently arbitrary. snapshots[0].Value.ShouldBe(99.0); + snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); + } + + /// + /// R06:S-1. A key-value tag pointed at a history table (many rows per key) used to pull every + /// matching row into memory on every poll. The plan now asks for one row more than it has keys; a full + /// result set proves the selector is ambiguous, so every member publishes a configuration error and a + /// (rate-limited) warning names the table. The positive control: the same tag against a clean source + /// reads Good. + /// + [Fact] + public async Task ReadAsync_keyValueRowCapExceeded_publishesAConfigurationError_forEveryMember() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var clean = Reader(fixture, logger, KvTag("Speed", SqlitePollFixture.PresentKey)); + (await clean.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.Good); + logger.Entries.ShouldBeEmpty(); + + for (var i = 0; i < 50; i++) + { + fixture.Execute(string.Create(CultureInfo.InvariantCulture, + $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + + $"VALUES ('{SqlitePollFixture.PresentKey}', {i}, '2026-07-24T11:00:00Z')")); + } + + var reader = Reader(fixture, logger, KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Speed2", SqlitePollFixture.PresentKey)); + var snapshots = await reader.ReadAsync(["Speed", "Speed2"], CancellationToken.None); + + snapshots.ShouldAllBe(x => x.StatusCode == SqlStatusCodes.BadConfigurationError); + var warning = logger.Entries.ShouldHaveSingleItem(); + warning.Level.ShouldBe(LogLevel.Warning); + warning.Message.ShouldContain(SqlitePollFixture.KeyValueTable); + } + + /// + /// The client-side half of the cap: a dialect whose limit fragments are empty (a provider that cannot + /// spell one) must still stop reading at cap + 1 and report the same configuration error. + /// + [Fact] + public async Task ReadAsync_aDialectThatEmitsNoLimit_isStillCappedClientSide() + { + using var fixture = new SqlitePollFixture(); + AddSecondRowForPresentStation(fixture); + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new NoLimitDialect(), + commandTimeout: CommandTimeout, operationTimeout: OperationTimeout, + maxConcurrentGroups: 4, nullIsBad: false, + resolve: Resolver(WideTag("Oven", "oven_temp", selectorValue: SqlitePollFixture.PresentStation))); + + var snapshots = await reader.ReadAsync(["Oven"], CancellationToken.None); + + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + } + + /// SQLite with the row-limit fragments blanked — the "provider cannot spell a limit" shape. + private sealed class NoLimitDialect : ISqlDialect + { + private readonly SqliteDialect _inner = new(); + public SqlProvider Provider => _inner.Provider; + public System.Data.Common.DbProviderFactory Factory => _inner.Factory; + public int MaxBindParameters => _inner.MaxBindParameters; + public string LivenessSql => _inner.LivenessSql; + public string SingleRowLimitPrefix => _inner.SingleRowLimitPrefix; + public string SingleRowLimitSuffix => _inner.SingleRowLimitSuffix; + public string RowLimitPrefix(int rows) => string.Empty; + public string RowLimitSuffix(int rows) => string.Empty; + public string ListSchemasSql => _inner.ListSchemasSql; + public string DefaultSchemaSql => _inner.DefaultSchemaSql; + public string ListTablesSql => _inner.ListTablesSql; + public string ListColumnsSql => _inner.ListColumnsSql; + public string QuoteIdentifier(string ident) => _inner.QuoteIdentifier(ident); + public string ParameterMarker(string baseName) => _inner.ParameterMarker(baseName); + public string ParameterBindName(string baseName) => _inner.ParameterBindName(baseName); + public void ConfigureCommand(System.Data.Common.DbCommand command) => _inner.ConfigureCommand(command); + public void ConfigureParameter(System.Data.Common.DbParameter parameter, object? value) => ((ISqlDialect)_inner).ConfigureParameter(parameter, value); + public DriverDataType MapColumnType(string sqlDataType) => _inner.MapColumnType(sqlDataType); } // ---- the frozen-peer contract ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs index 901382f1..912a8ee6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlServerDialectTests.cs @@ -13,6 +13,15 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests; /// public class SqlServerDialectTests { + [Fact] + public void RowLimit_isTopN_inThePrefixPosition() + { + var d = new SqlServerDialect(); + d.RowLimitPrefix(3).ShouldBe("TOP (3) "); + d.RowLimitSuffix(3).ShouldBe(""); + Should.Throw(() => d.RowLimitPrefix(0)); + } + [Fact] public void QuoteIdentifier_bracketsAndEscapesEmbeddedBrackets() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs index 62b9f3f4..20dded60 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqliteDialect.cs @@ -87,6 +87,20 @@ public sealed class SqliteDialect : ISqlDialect /// public string SingleRowLimitSuffix => " LIMIT 1"; + /// + public string RowLimitPrefix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return string.Empty; + } + + /// + public string RowLimitSuffix(int rows) + { + ArgumentOutOfRangeException.ThrowIfLessThan(rows, 1); + return " LIMIT " + rows.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + /// /// SQLite has no schema namespace, so the schema level is the single literal /// — enough to give the browser a real root to expand. From ab29b76be0feddc39844e8146230920dc670e543 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:00:25 -0400 Subject: [PATCH 12/20] perf(sql): rate-limit per-poll warnings and cache poll plans per reference list (R06:S-6, S-7) S-6: a frozen group's BadTimeout warning and a failing group's query-error warning (with SQL text) logged every poll per group indefinitely; both are now rate-limited per plan (once a minute), and an uncoercible cell warns once per tag instead of once per poll. Every poll still publishes the Bad status. S-7: plans and the slot map were rebuilt every poll although a poll loop passes the same reference list each cycle. They are cached per list instance (weakly keyed) and rebuilt only when a reference resolves to a different definition instance, which a catalog-gate swap produces. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlPollReader.cs | 183 ++++++++++++++---- .../SqlPollReaderTests.cs | 120 ++++++++++++ 2 files changed, 264 insertions(+), 39 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index 25e70a39..fdb22ff0 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -1,5 +1,7 @@ +using System.Collections.Concurrent; using System.Data.Common; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Globalization; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -68,6 +70,27 @@ public sealed class SqlPollReader : IDisposable private long _lastContractWarningTicks; + /// + /// Per-key last-warned instant for the per-poll warnings (R06:S-6): a frozen group's BadTimeout and a + /// failing group's query error used to log every poll, per group, indefinitely — 20 groups at 1 s was + /// 20 lines a second carrying SQL text. Keyed by the plan's SQL text, so bounded by the plan count. + /// + private readonly ConcurrentDictionary _lastWarnedTicks = new(StringComparer.Ordinal); + + /// Tags whose uncoercible cell has been warned about — once per tag per reader (the refusal rule). + private readonly ConcurrentDictionary _coerceWarned = new(StringComparer.Ordinal); + + /// + /// The resolved definitions, slot map and plans for each reference list a caller polls (R06:S-7). A + /// poll loop passes the SAME list instance every cycle, so it is planned once instead of every poll. + /// Weakly keyed: an unsubscribed list (and its entry) is collected with it. Invalidation is structural — + /// each entry remembers the definition INSTANCES it was built from, and any reference resolving to a + /// different instance (a catalog-gate swap) rebuilds it. + /// + private readonly ConditionalWeakTable, PollShape> _shapes = new(); + + private int _plansBuilt; + /// Initializes a new instance of the class. /// /// Creates the provider's connections. Passed explicitly rather than taken from @@ -182,28 +205,18 @@ public sealed class SqlPollReader : IDisposable if (results.Length == 0) return new SqlPollResult(results, 0); var readAt = DateTime.UtcNow; + var shape = GetShape(fullReferences); - // Resolve first. A miss is this tag's own problem (BadNodeIdUnknown) and must not reach the planner, - // whose members are the sole thing the slice-back walks. - var definitions = new List(fullReferences.Count); - var slots = new Dictionary>(); - for (var i = 0; i < fullReferences.Count; i++) + // A miss is this tag's own problem (BadNodeIdUnknown) and never reaches the planner, whose members + // are the sole thing the slice-back walks. + for (var i = 0; i < shape.Resolved.Length; i++) { - var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; - if (definition is null) - { + if (shape.Resolved[i] is null) results[i] = new DataValueSnapshot(null, SqlStatusCodes.BadNodeIdUnknown, null, readAt); - continue; - } - - definitions.Add(definition); - if (!slots.TryGetValue(definition, out var positions)) - slots[definition] = positions = []; - positions.Add(i); } - var rowCapExceeded = definitions.Count > 0 - ? await ReadGroupsAsync(definitions, slots, results, readAt, cancellationToken).ConfigureAwait(false) + var rowCapExceeded = shape.Definitions.Count > 0 + ? await ReadGroupsAsync(shape, results, readAt, cancellationToken).ConfigureAwait(false) : 0; // Fail-safe. Nothing above should leave a hole, but "N in, N out" is a contract the caller indexes @@ -214,30 +227,93 @@ public sealed class SqlPollReader : IDisposable return new SqlPollResult(results, rowCapExceeded); } + /// How many times a reference list has been (re)planned. Test observation of the S-7 cache. + internal int PlansBuiltForTests => Volatile.Read(ref _plansBuilt); + /// - /// Plans the resolved definitions, runs every group concurrently, and slices the results back. - /// Returns how many groups came back over their row cap. + /// Returns the cached shape for , rebuilding it when any reference now + /// resolves to a different definition instance than the one it was built from. /// - private async Task ReadGroupsAsync( - List definitions, - Dictionary> slots, - DataValueSnapshot[] results, - DateTime readAt, - CancellationToken cancellationToken) + private PollShape GetShape(IReadOnlyList fullReferences) { - IReadOnlyList plans; + if (_shapes.TryGetValue(fullReferences, out var cached) && cached.Resolved.Length == fullReferences.Count) + { + var current = true; + for (var i = 0; i < fullReferences.Count && current; i++) + { + var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; + current = ReferenceEquals(definition, cached.Resolved[i]); + } + + if (current) return cached; + } + + var shape = BuildShape(fullReferences); + _shapes.AddOrUpdate(fullReferences, shape); + return shape; + } + + /// Resolves, slots and plans one reference list. + private PollShape BuildShape(IReadOnlyList fullReferences) + { + Interlocked.Increment(ref _plansBuilt); + + var resolved = new SqlTagDefinition?[fullReferences.Count]; + var definitions = new List(fullReferences.Count); + var slots = new Dictionary>(); + for (var i = 0; i < fullReferences.Count; i++) + { + var definition = fullReferences[i] is { } reference ? _resolve(reference) : null; + resolved[i] = definition; + if (definition is null) continue; + + definitions.Add(definition); + if (!slots.TryGetValue(definition, out var positions)) + slots[definition] = positions = []; + positions.Add(i); + } + + if (definitions.Count == 0) return new PollShape(resolved, definitions, slots, [], PlanningFailed: false); + try { - plans = SqlGroupPlanner.Plan(definitions, _dialect); + return new PollShape(resolved, definitions, slots, SqlGroupPlanner.Plan(definitions, _dialect), PlanningFailed: false); } catch (Exception ex) when (ex is ArgumentException or NotSupportedException) { // A definition with a broken invariant — SqlEquipmentTagParser rejects all of these up front, so // reaching here means a definition was built past the parser. It is an authoring fault, not a // database fault: Bad-code the planned tags rather than throwing and triggering connection backoff. + // Logged once per (re)plan, which the cache makes once per shape rather than once per poll. _logger.LogError(ex, "Sql poll planning failed for {Count} tag(s); publishing BadConfigurationError.", definitions.Count); - foreach (var definition in definitions) + return new PollShape(resolved, definitions, slots, [], PlanningFailed: true); + } + } + + /// One reference list's resolved definitions, slot map and plans — immutable once built. + private sealed record PollShape( + SqlTagDefinition?[] Resolved, + List Definitions, + Dictionary> Slots, + IReadOnlyList Plans, + bool PlanningFailed); + + /// + /// Runs every planned group concurrently and slices the results back. Returns how many groups came + /// back over their row cap. + /// + private async Task ReadGroupsAsync( + PollShape shape, + DataValueSnapshot[] results, + DateTime readAt, + CancellationToken cancellationToken) + { + var slots = shape.Slots; + var plans = shape.Plans; + if (shape.PlanningFailed) + { + foreach (var definition in shape.Definitions) { if (!slots.TryGetValue(definition, out var positions)) continue; var snapshot = new DataValueSnapshot(null, SqlStatusCodes.BadConfigurationError, null, readAt); @@ -429,11 +505,14 @@ public sealed class SqlPollReader : IDisposable { // The table is named for the same reason the contract warnings name it: under a partial outage this // line repeats every poll, and "which source is frozen" is the only question the operator has. - _logger.LogWarning( - "Sql poll group ({Model}, '{Table}', {Members} tag(s)) exceeded the {Timeout} ms operation " + - "timeout; publishing BadTimeout.", - plan.Model, plan.Members[0].Table, plan.Members.Count, - (int)_operationTimeout.TotalMilliseconds); + if (ShouldWarn("timeout:" + plan.SqlText)) + { + _logger.LogWarning( + "Sql poll group ({Model}, '{Table}', {Members} tag(s)) exceeded the {Timeout} ms operation " + + "timeout; publishing BadTimeout. Repeats for this group are suppressed for {Interval} s.", + plan.Model, plan.Members[0].Table, plan.Members.Count, + (int)_operationTimeout.TotalMilliseconds, (int)ContractWarningInterval.TotalSeconds); + } AbandonedTask.Detach(work); return GroupResult.TimedOut; @@ -495,9 +574,13 @@ public sealed class SqlPollReader : IDisposable { // The connection opened, so the server is up — this query is what failed (an identifier that // is not in the catalog, a lock, a permission). That is this group's problem alone. - _logger.LogWarning( - ex, "Sql poll group ({Model}, {Members} tag(s)) failed: {Sql}", - plan.Model, plan.Members.Count, plan.SqlText); + if (ShouldWarn("failed:" + plan.SqlText)) + { + _logger.LogWarning( + ex, "Sql poll group ({Model}, {Members} tag(s)) failed: {Sql}. Repeats for this group are " + + "suppressed for {Interval} s.", + plan.Model, plan.Members.Count, plan.SqlText, (int)ContractWarningInterval.TotalSeconds); + } return GroupResult.Failed; } } @@ -746,9 +829,14 @@ public sealed class SqlPollReader : IDisposable var target = member.DeclaredType ?? _dialect.MapColumnType(outcome.TypeNames[valueAt]); if (!TryCoerce(cell, target, out var value)) { - _logger.LogWarning( - "Sql tag '{Tag}' could not coerce a {Actual} cell to its {Declared} type.", - member.Name, cell.GetType().Name, target); + // A per-tag refusal: warned ONCE per tag (per reader), never per poll (R06:S-6). + if (_coerceWarned.TryAdd(member.Name, 0)) + { + _logger.LogWarning( + "Sql tag '{Tag}' could not coerce a {Actual} cell to its {Declared} type; it publishes " + + "BadTypeMismatch (warned once per tag).", + member.Name, cell.GetType().Name, target); + } return new DataValueSnapshot(null, SqlStatusCodes.BadTypeMismatch, sourceTimestamp, readAt); } @@ -815,6 +903,23 @@ public sealed class SqlPollReader : IDisposable Convert.ToDateTime(cell, CultureInfo.InvariantCulture), DateTimeKind.Utc), }; + /// Rate-limits one per-poll warning by key; true at most once per . + private bool ShouldWarn(string key) + { + var now = Stopwatch.GetTimestamp(); + while (true) + { + if (!_lastWarnedTicks.TryGetValue(key, out var last)) + { + if (_lastWarnedTicks.TryAdd(key, now)) return true; + continue; + } + + if (Stopwatch.GetElapsedTime(last, now) < ContractWarningInterval) return false; + if (_lastWarnedTicks.TryUpdate(key, now, last)) return true; + } + } + /// Rate-limits the source-contract warning so a misconfigured table cannot flood the log. private bool ShouldWarnAboutContract() { diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index 4181449c..d26bd0b6 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -605,6 +605,126 @@ public sealed class SqlPollReaderTests dialect.Factory, " ", dialect, CommandTimeout, OperationTimeout, 4, false, _ => null)); } + // ---- R06:S-7: plans are cached per subscription reference list ---- + + /// + /// A poll loop hands the reader the SAME reference list every cycle; re-planning it (and rebuilding the + /// slot dictionary keyed by a 12-field record) every poll was pure per-poll garbage. Planned once, + /// and the cached plan still reads correct values every time — the positive control. + /// + [Fact] + public async Task ReadAsync_theSameReferenceList_isPlannedOnce() + { + using var fixture = new SqlitePollFixture(); + var reader = Reader(fixture, + KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Pressure", SqlitePollFixture.SecondPresentKey)); + IReadOnlyList refs = ["Speed", "Pressure"]; + + for (var i = 0; i < 3; i++) + { + var snapshots = await reader.ReadAsync(refs, CancellationToken.None); + snapshots[0].Value.ShouldBe(SqlitePollFixture.PresentValue); + snapshots[1].Value.ShouldBe(SqlitePollFixture.SecondPresentValue); + } + + reader.PlansBuiltForTests.ShouldBe(1); + } + + /// + /// The cache is invalidated by the definitions themselves: a catalog-gate swap (or any re-resolve) + /// hands back different definition instances, and the next poll re-plans against them. + /// + [Fact] + public async Task ReadAsync_whenAReferenceResolvesToADifferentDefinition_rePlans() + { + using var fixture = new SqlitePollFixture(); + var table = new Dictionary(StringComparer.Ordinal) + { + ["Speed"] = KvTag("Speed", SqlitePollFixture.PresentKey), + }; + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: CommandTimeout, operationTimeout: OperationTimeout, + maxConcurrentGroups: 4, nullIsBad: false, resolve: r => table.GetValueOrDefault(r)); + IReadOnlyList refs = ["Speed"]; + + (await reader.ReadAsync(refs, CancellationToken.None))[0].Value.ShouldBe(SqlitePollFixture.PresentValue); + table["Speed"] = KvTag("Speed", SqlitePollFixture.SecondPresentKey); // re-pointed by a new resolve + + (await reader.ReadAsync(refs, CancellationToken.None))[0].Value.ShouldBe(SqlitePollFixture.SecondPresentValue); + reader.PlansBuiltForTests.ShouldBe(2); + } + + // ---- R06:S-6: per-poll warnings are rate-limited ---- + + /// + /// A dropped table polled every second used to log a warning — with the full SQL text — on every + /// poll, per group, forever. One per minute per plan now; every poll still publishes the Bad status. + /// The first poll's warning is the positive control. + /// + [Fact] + public async Task ReadAsync_aPersistentlyFailingGroup_warnsOncePerInterval() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = Reader(fixture, logger, + new SqlTagDefinition("Gone", SqlTagModel.KeyValue, "NoSuchTable", + KeyColumn: SqlitePollFixture.KeyColumn, KeyValue: "k", ValueColumn: SqlitePollFixture.ValueColumn)); + + for (var i = 0; i < 3; i++) + { + (await reader.ReadAsync(["Gone"], CancellationToken.None))[0] + .StatusCode.ShouldBe(SqlStatusCodes.BadCommunicationError); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning).ShouldBe(1); + } + + /// A cell that will not coerce is a per-tag refusal: one warning per tag, not one per poll. + [Fact] + public async Task ReadAsync_anUncoercibleCell_warnsOncePerTag() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = Reader(fixture, logger, + KvTag("Speed", SqlitePollFixture.PresentKey, declaredType: DriverDataType.DateTime)); + + for (var i = 0; i < 3; i++) + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0] + .StatusCode.ShouldBe(SqlStatusCodes.BadTypeMismatch); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning).ShouldBe(1); + } + + /// A frozen group's BadTimeout warning is rate-limited too; both polls still time out. + [Fact] + public async Task ReadAsync_aRepeatedlyTimedOutGroup_warnsOncePerInterval() + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + var reader = new SqlPollReader(fixture.Factory, fixture.ConnectionString, new SqliteDialect(), + commandTimeout: TimeSpan.FromSeconds(30), operationTimeout: TimeSpan.FromMilliseconds(500), + maxConcurrentGroups: 4, nullIsBad: false, + resolve: Resolver(KvTag("Speed", SqlitePollFixture.PresentKey)), logger: logger); + + var locker = fixture.OpenNewConnection(); + Execute(locker, "BEGIN EXCLUSIVE"); + try + { + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + (await reader.ReadAsync(["Speed"], CancellationToken.None))[0].StatusCode.ShouldBe(SqlStatusCodes.BadTimeout); + } + finally + { + Execute(locker, "ROLLBACK"); + locker.Dispose(); + } + + logger.Entries.Count(e => e.Level == LogLevel.Warning && e.Message.Contains("operation timeout")) + .ShouldBe(1); + } + // ---- R06:S-3: the concurrency gate is released ---- /// An idle reader releases its gate on Dispose, and a disposed reader starts no new group. From a7d21d5a659fc905c612086192a1e27f4513cc5c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:03:23 -0400 Subject: [PATCH 13/20] feat(sql): IBatchSubscribable and lifecycle-generation-stamped health (R02:P1, R06:S-9) R02:P1: SqlDriver runs PollGroupEngine, so it now implements IBatchSubscribable and raises one OnDataChangeBatch per poll burst (DriverInstanceActor forwards one message per cycle instead of one per tag). S-9: InitializeAsync/ShutdownAsync take a lifecycle generation; their health writes apply only while current, and a poll that started under an older generation (one that outlived the engine's teardown wait) no longer writes its verdict onto the new configuration. Poll-driven updates can no longer end Initializing - only the initialize's own final write does. Lifecycle writes now preserve LastSuccessfulRead. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 96 ++++++++++++---- .../SqlDriverTests.cs | 104 ++++++++++++++++++ 2 files changed, 179 insertions(+), 21 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 4a8936c9..5492c293 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -28,7 +28,7 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// manufacture an exception to force backoff. /// public sealed class SqlDriver - : IDriver, ITagDiscovery, IReadable, ISubscribable, IHostConnectivityProbe, ISkippedTagReporter, IAsyncDisposable + : IDriver, ITagDiscovery, IReadable, IBatchSubscribable, IHostConnectivityProbe, ISkippedTagReporter, IAsyncDisposable { /// Authored tags this driver could not build into its poll table. private readonly SkippedTagLedger _skippedTags = new(); @@ -156,6 +156,16 @@ public sealed class SqlDriver private DriverHealth _health = new(DriverState.Unknown, null, null); + /// Guards the lifecycle-generation bump against the generation-checked lifecycle health write. + private readonly object _healthLock = new(); + + /// + /// Bumped by every and . A lifecycle write from + /// a superseded call, and a poll that STARTED under an older generation (a stale poll outliving the + /// engine's 5 s teardown wait, R06:S-9), can no longer overwrite the newer state. + /// + private long _lifecycleGeneration; + /// /// The exception instance has already turned into a health verdict, held only /// until — which the poll engine calls with that same object — has @@ -173,6 +183,9 @@ public sealed class SqlDriver /// Occurs when a subscribed tag's value or quality changes. public event EventHandler? OnDataChange; + /// Occurs once per poll cycle that produced at least one change, carrying every change of that cycle. + public event EventHandler? OnDataChangeBatch; + /// Occurs when the database endpoint transitions between reachable and unreachable. public event EventHandler? OnHostStatusChanged; @@ -230,6 +243,9 @@ public sealed class SqlDriver OnDataChange?.Invoke(this, new DataChangeEventArgs(handle, tagRef, snapshot)), onError: HandlePollError, backoffCap: PollBackoffCap, + // R02:P1 — one batch per poll burst, so DriverInstanceActor forwards one message per cycle instead + // of one per tag (it subscribes exactly one of the two events). + onBatch: (handle, items) => OnDataChangeBatch?.Invoke(this, new DataChangeBatchEventArgs(handle, items)), // The engine owns no Meter (Core.Abstractions takes no metrics dependency), so this // closure IS the production consumer of its overrun signal. onOverrun: _ => OtOpcUaTelemetry.PollOverruns.Add(1, @@ -352,7 +368,9 @@ public sealed class SqlDriver // identifier. public async Task InitializeAsync(string driverConfigJson, CancellationToken cancellationToken) { - WriteHealth(new DriverHealth(DriverState.Initializing, null, null)); + var generation = BeginLifecycle(); + // LastSuccessfulRead is PRESERVED across the lifecycle states — it is the watchdog's evidence clock. + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Initializing, ReadHealth().LastSuccessfulRead, null)); // adopt a changed config BEFORE the tag table is built, so the table and the connection it // will be polled over come from the same revision. ApplyBinding takes the options, dialect and @@ -374,7 +392,7 @@ public sealed class SqlDriver catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { // The caller tore the initialization down; not a database verdict. - WriteHealth(new DriverHealth(DriverState.Unknown, null, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, ReadHealth().LastSuccessfulRead, null)); throw; } catch (Exception ex) @@ -390,8 +408,8 @@ public sealed class SqlDriver $"Check the database is reachable and the connection string is valid."; _logger.LogError(ex, "Sql driver {DriverInstanceId} liveness check against {Endpoint} failed.", _driverInstanceId, Endpoint); - WriteHealth(new DriverHealth(DriverState.Faulted, null, message)); - TransitionHostTo(HostState.Stopped); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, message))) + TransitionHostTo(HostState.Stopped); throw new InvalidOperationException(message, ex); } @@ -404,7 +422,7 @@ public sealed class SqlDriver } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { - WriteHealth(new DriverHealth(DriverState.Unknown, null, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, ReadHealth().LastSuccessfulRead, null)); throw; } catch (Exception ex) @@ -420,13 +438,13 @@ public sealed class SqlDriver _logger.LogError(ex, "Sql driver {DriverInstanceId} catalog validation against {Endpoint} failed.", _driverInstanceId, Endpoint); - WriteHealth(new DriverHealth(DriverState.Faulted, null, message)); - TransitionHostTo(HostState.Stopped); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, message))) + TransitionHostTo(HostState.Stopped); throw new InvalidOperationException(message, ex); } - WriteHealth(new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null)); - TransitionHostTo(HostState.Running); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Healthy, DateTime.UtcNow, null))) + TransitionHostTo(HostState.Running); _logger.LogInformation( "Sql driver {DriverInstanceId} initialized against {Endpoint} with {TagCount} authored tag(s).", _driverInstanceId, Endpoint, Tags.Count); @@ -442,9 +460,10 @@ public sealed class SqlDriver /// public async Task ShutdownAsync(CancellationToken cancellationToken) { + var generation = BeginLifecycle(); var lastRead = ReadHealth().LastSuccessfulRead; await TeardownAsync().ConfigureAwait(false); - WriteHealth(new DriverHealth(DriverState.Unknown, lastRead, null)); + WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Unknown, lastRead, null)); } /// @@ -517,10 +536,13 @@ public sealed class SqlDriver // unreachable" and degrade a perfectly healthy driver. ArgumentNullException.ThrowIfNull(fullReferences); + // The generation this read started under: if a Shutdown/Initialize lands while it is in flight, its + // verdict describes a superseded configuration and must not touch the newer health (R06:S-9). + var generation = Interlocked.Read(ref _lifecycleGeneration); try { var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); - ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups); + ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups, generation); return result.Snapshots; } catch (OperationCanceledException) @@ -535,8 +557,12 @@ public sealed class SqlDriver _driverInstanceId, Endpoint); // Type name only — never ex.Message (see InitializeAsync for why); the full exception is already // on the log sink via the LogWarning exception parameter above. - Degrade($"Sql driver '{_driverInstanceId}' could not reach {Endpoint}: {ex.GetType().Name}."); - TransitionHostTo(HostState.Stopped); + if (IsCurrentLifecycle(generation)) + { + Degrade($"Sql driver '{_driverInstanceId}' could not reach {Endpoint}: {ex.GetType().Name}."); + TransitionHostTo(HostState.Stopped); + } + // Mark THIS exception object as already classified before it goes back to the poll engine, which // routes it to HandlePollError next. The mark is what makes the deferral there exact — see that // method for why inferring it from the exception's type was not. @@ -614,9 +640,15 @@ public sealed class SqlDriver /// (the planner mints it before any I/O), so the count is what makes them count as the database /// answering. /// - internal void ObservePollOutcome(IReadOnlyList snapshots, int answeredRefusals = 0) + /// + /// The lifecycle generation the poll started under; a superseded one is ignored outright. Null (tests) + /// means "current". + /// + internal void ObservePollOutcome( + IReadOnlyList snapshots, int answeredRefusals = 0, long? generation = null) { if (snapshots.Count == 0) return; + if (generation is { } started && !IsCurrentLifecycle(started)) return; var good = 0; var unreachable = 0; @@ -800,7 +832,11 @@ public sealed class SqlDriver /// /// Builds the snapshot to publish from the current one. private void UpdateHealthUnlessFaulted(Func project) - => UpdateHealth(current => current.State == DriverState.Faulted ? null : project(current)); + // Initializing too: only the initialize's own final write may end it — a poll landing mid-init + // (a re-init without shutdown keeps the engine running) used to flip Initializing → Healthy. + => UpdateHealth(current => current.State is DriverState.Faulted or DriverState.Initializing + ? null + : project(current)); /// /// The one read-modify-write of the health field, as a compare-and-swap retry loop. @@ -831,13 +867,31 @@ public sealed class SqlDriver /// Barrier-protected read of the multi-threaded health field. private DriverHealth ReadHealth() => Volatile.Read(ref _health); + /// Starts a lifecycle operation and returns its generation; every later one supersedes it. + private long BeginLifecycle() + { + lock (_healthLock) return ++_lifecycleGeneration; + } + + /// Whether is still the newest lifecycle operation. + private bool IsCurrentLifecycle(long generation) => Interlocked.Read(ref _lifecycleGeneration) == generation; + + /// The current lifecycle generation. Test observation of the S-9 guard. + internal long LifecycleGenerationForTests => Interlocked.Read(ref _lifecycleGeneration); + /// - /// Unconditional publish of a new health snapshot, for the lifecycle transitions that are - /// last-writer-wins by definition (Initialize / Shutdown). Everything driven by a poll goes through - /// instead, because it has to read the current snapshot first. + /// A lifecycle write that applies only while is current — atomic against + /// , so a superseded initialize/shutdown never overwrites a newer one. /// - /// The health snapshot to publish. - private void WriteHealth(DriverHealth value) => Volatile.Write(ref _health, value); + private bool WriteHealthIfCurrent(long generation, DriverHealth value) + { + lock (_healthLock) + { + if (_lifecycleGeneration != generation) return false; + Volatile.Write(ref _health, value); + return true; + } + } /// Records a host-state change and raises — on transitions only. private void TransitionHostTo(HostState newState) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 6dd21ce9..320a0a0f 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -385,6 +385,85 @@ public sealed class SqlDriverTests driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull().ShouldBeGreaterThan(before); } + /// + /// R06:S-9 / lifecycle generations. A poll that started before a Shutdown → Initialize cycle + /// (it can outlive the engine's 5 s teardown wait) must not write its verdict onto the NEW + /// configuration's health. Control: the same BadTimeout verdict under the current generation degrades. + /// + [Fact] + public async Task ObservePollOutcome_fromASupersededGeneration_isIgnored() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + var stale = driver.LifecycleGenerationForTests; + + await driver.ShutdownAsync(CancellationToken.None); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + var timedOut = new DataValueSnapshot(null, SqlStatusCodes.BadTimeout, null, DateTime.UtcNow); + + driver.ObservePollOutcome([timedOut], generation: stale); + driver.GetHealth().State.ShouldBe(DriverState.Healthy, "a superseded poll must not degrade the new init"); + + driver.ObservePollOutcome([timedOut], generation: driver.LifecycleGenerationForTests); + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + } + + /// + /// Only the initialize's own final write ends Initializing: a Good poll landing mid-init used to + /// flip it to Healthy before liveness had even answered. Control: once the init completes it is Healthy. + /// + [Fact] + public async Task APollLandingMidInitialize_cannotEndInitializing() + { + using var fixture = new SqlitePollFixture(); + var factory = new GatedFactory(); + await using var driver = NewDriver(fixture, fixture.ConnectionString, CapturingLogger.Null, factory, + KvEntry("Speed", SqlitePollFixture.PresentKey)); + + factory.Arm(); + var init = driver.InitializeAsync(ConfigJson, CancellationToken.None); + (await factory.Entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken)).ShouldBeTrue(); + driver.GetHealth().State.ShouldBe(DriverState.Initializing); + + driver.ObservePollOutcome([new DataValueSnapshot(42.0, SqlStatusCodes.Good, DateTime.UtcNow, DateTime.UtcNow)]); + driver.GetHealth().State.ShouldBe(DriverState.Initializing); + + factory.Release(); + await init.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// A SQLite factory whose next connection creation blocks until released — parks an initialize. + private sealed class GatedFactory : DbProviderFactory + { + private readonly ManualResetEventSlim _release = new(initialState: true); + private volatile bool _armed; + + public TaskCompletionSource Entered { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void Arm() + { + _release.Reset(); + _armed = true; + } + + public void Release() => _release.Set(); + + public override DbConnection CreateConnection() + { + if (_armed) + { + _armed = false; + Entered.TrySetResult(true); + _release.Wait(TimeSpan.FromSeconds(30)); + } + + return Microsoft.Data.Sqlite.SqliteFactory.Instance.CreateConnection()!; + } + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] @@ -474,6 +553,31 @@ public sealed class SqlDriverTests Volatile.Read(ref changes).ShouldBe(afterUnsubscribe); } + /// + /// R02:P1. The driver runs PollGroupEngine, so it raises one batch per poll burst: + /// DriverInstanceActor then forwards one message per cycle instead of one per tag. Both tags of + /// the first (force-raised) poll arrive in the SAME batch, on the subscription's handle. + /// + [Fact] + public async Task Subscribe_raisesOneBatchPerPollBurst() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, + KvEntry("Speed", SqlitePollFixture.PresentKey), KvEntry("Pressure", SqlitePollFixture.SecondPresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + + var batchable = driver.ShouldBeAssignableTo(); + var first = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + batchable!.OnDataChangeBatch += (_, args) => first.TrySetResult(args); + + var handle = await driver.SubscribeAsync(["Speed", "Pressure"], TimeSpan.FromMilliseconds(100), CancellationToken.None); + + var batch = await first.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + batch.SubscriptionHandle.ShouldBe(handle); + batch.Items.Select(i => i.FullReference).OrderBy(r => r, StringComparer.Ordinal).ShouldBe(["Pressure", "Speed"]); + batch.Items.Single(i => i.FullReference == "Speed").Snapshot.Value.ShouldBe(SqlitePollFixture.PresentValue); + } + // ---- disposal ---- [Fact] From 2a28c06fc790fe5d985e27e427a7879af3cab03c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:04:59 -0400 Subject: [PATCH 14/20] fix(sql): a skipped tag reports the deploy gate's reason, pinned for the deferred Query model (R06:S-4) A model:Query tag reached the runtime skip ledger as a generic 'did not map'. The ledger reason (and the skip warning) now comes from SqlTagDefinitionFactory.Inspect - the deploy gate's own explanation over the same parser - so /hosts names the deferral. A test pins that the deploy gate names Query through the parser. Making it a hard deploy error in Warn mode is a ControlPlane change (follow-up); MySql/Odbc and allowWrites remain feature issues. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 11 +++++++--- .../SqlDriverTests.cs | 20 +++++++++++++++++++ .../SqlEquipmentTagParserTests.cs | 15 ++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 5492c293..7fbd99d7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -929,10 +929,15 @@ public sealed class SqlDriver table[entry.RawPath] = definition; else { - blobSkips.Add(new SkippedTag(entry.RawPath, "TagConfig did not map to a definition")); + // The reason is the DEPLOY GATE's own explanation (R06:S-4) — SqlTagDefinitionFactory.Inspect + // asks this same parser and then says why — so /hosts names e.g. the deferred Query model + // instead of a generic "did not map" for a tag the gate already warned about. + var reason = SqlTagDefinitionFactory.Inspect(entry.TagConfig).FirstOrDefault() + ?? "TagConfig did not map to a definition"; + blobSkips.Add(new SkippedTag(entry.RawPath, reason)); _logger.LogWarning( - "Sql tag config did not map to a definition; skipping. Driver={DriverInstanceId} RawPath={RawPath}", - _driverInstanceId, entry.RawPath); + "Sql tag config did not map to a definition; skipping. Driver={DriverInstanceId} RawPath={RawPath} Reason={Reason}", + _driverInstanceId, entry.RawPath, reason); } } catch (Exception ex) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 320a0a0f..1eec94e5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -464,6 +464,26 @@ public sealed class SqlDriverTests } } + /// + /// R06:S-4. A skipped tag's ledger reason (what /hosts shows) is the deploy gate's own explanation + /// — for a Query tag, that the model is deferred — instead of a generic "did not map". Control: a + /// well-formed tag is not skipped at all. + /// + [Fact] + public async Task ADeferredQueryTag_isSkipped_withTheGatesReason() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, + KvEntry("Speed", SqlitePollFixture.PresentKey), + new RawTagEntry("Q", """{"driver":"Sql","model":"Query","table":"T","query":"SELECT 1"}""", WriteIdempotent: false)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + + var report = ((ISkippedTagReporter)driver).GetSkippedTags().ShouldNotBeNull(); + var skipped = report.Sample.ShouldHaveSingleItem(); + skipped.RawPath.ShouldBe("Q"); + skipped.Reason.ShouldContain("Query"); + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs index 4734ee30..3433a4af 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlEquipmentTagParserTests.cs @@ -232,6 +232,21 @@ public class SqlEquipmentTagParserTests SqlEquipmentTagParser.TryParse(json, "raw", out _).ShouldBeFalse(); } + /// + /// R06:S-4. The deploy gate (EquipmentTagConfigInspector) reaches this through the parser: + /// a Query tag draws a warning naming the deferral (a rejection under TagConfigValidation=Error). + /// Control: a served key-value blob draws nothing. + /// + [Fact] + public void Inspect_theDeferredQueryModel_isNamedAtTheDeployGate() + { + SqlTagDefinitionFactory.Inspect("""{"driver":"Sql","model":"Query","table":"T","query":"SELECT 1"}""") + .ShouldHaveSingleItem().ShouldContain("Query"); + SqlTagDefinitionFactory.Inspect( + """{"driver":"Sql","model":"KeyValue","table":"T","keyColumn":"k","keyValue":"v","valueColumn":"c"}""") + .ShouldBeEmpty(); + } + [Theory] [InlineData("""{"driver":"Sql","model":"KeyValue","keyColumn":"k","keyValue":"v","valueColumn":"c"}""")] [InlineData("""{"driver":"Sql","model":"KeyValue","table":"","keyColumn":"k","keyValue":"v","valueColumn":"c"}""")] From a7506e7ed4fc73c5bb314fa418e2c4e824b12141 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:10:12 -0400 Subject: [PATCH 15/20] test(sql): live SQL Server gate for the row cap (R06:S-1) An events table with 40 rows for one where-pair selector: the capped SELECT TOP (2) runs as real T-SQL, the ambiguous tag publishes BadConfigurationError, a single-row station reads Good through the same statement, and the driver stays Healthy. Live run 2026-10-22: SQL Server 22/22, Postgres 7/7 and Oracle 6/6 (LIMIT n / FETCH FIRST n ROWS ONLY after WHERE) all green on 10.100.0.35. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlServerReadTests.cs | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs index ebf2d212..ff6271bc 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs @@ -58,6 +58,64 @@ public sealed class SqlServerReadTests driver.GetHostStatuses().ShouldHaveSingleItem().State.ShouldBe(HostState.Running); } + // ---- R06:S-1: the row cap, on real T-SQL ---- + + /// + /// A where-pair selector pointed at an events table (40 rows for one station) used to pull every + /// matching row on every poll. The plan now emits SELECT TOP (2) … — proven valid T-SQL here — + /// and the second row makes the tag publish BadConfigurationError while the driver stays + /// Healthy (the server answered). The positive control: a station with exactly one row reads Good + /// through the same capped statement. + /// + [Fact] + public async Task ReadAsync_realSqlServer_anAmbiguousSelectorIsRowCappedToAConfigurationError() + { + SkipUnlessLive(); + const string table = "RowCapEvents"; + + await using (var arrange = await _sql.OpenInspectionConnectionAsync()) + { + await using var command = arrange.CreateCommand(); + command.CommandText = string.Create(CultureInfo.InvariantCulture, $""" + IF OBJECT_ID(N'{SqlPollServerFixture.Schema}.{table}') IS NOT NULL DROP TABLE {SqlPollServerFixture.Schema}.{table}; + CREATE TABLE {SqlPollServerFixture.Schema}.{table} ( + {SqlPollServerFixture.WideRowSelectorColumn} int NOT NULL, + oven_temp float NULL, + {SqlPollServerFixture.TimestampColumn} datetime2 NOT NULL); + INSERT INTO {SqlPollServerFixture.Schema}.{table} VALUES (1, 21.5, '2026-07-24T10:00:00'); + DECLARE @i int = 0; + WHILE @i < 40 + BEGIN + INSERT INTO {SqlPollServerFixture.Schema}.{table} VALUES (7, @i, DATEADD(second, @i, '2026-07-24T10:00:00')); + SET @i += 1; + END + """); + await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + try + { + await using var driver = await StartAsync( + WideRowWhere("Sql/Events/Ambiguous", "oven_temp", "7", table), + WideRowWhere("Sql/Events/Clean", "oven_temp", "1", table)); + + var snapshots = await driver.ReadAsync( + ["Sql/Events/Ambiguous", "Sql/Events/Clean"], TestContext.Current.CancellationToken); + + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.Good); + Convert.ToDouble(snapshots[1].Value).ShouldBe(21.5); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + finally + { + await using var cleanup = await _sql.OpenInspectionConnectionAsync(); + await using var drop = cleanup.CreateCommand(); + drop.CommandText = $"DROP TABLE IF EXISTS {SqlPollServerFixture.Schema}.{table}"; + await drop.ExecuteNonQueryAsync(CancellationToken.None); + } + } + // ---- key-value model ---- /// The plan's headline case: a seeded key-value row round-trips as a Good snapshot. From d77a538e179b77b02a9c720077c6d874205ea66b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:17:09 -0400 Subject: [PATCH 16/20] test(mqtt): pass the test cancellation token to the wedged-init lifecycle test Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../MqttDriverSessionLifecycleTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 6abfa293..14dbda13 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -267,7 +267,7 @@ public sealed class MqttDriverSessionLifecycleTests await release.Task; }; - var initialize = Task.Run(() => driver.InitializeAsync(Json(options), CancellationToken.None)); + var initialize = Task.Run(() => driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken)); await entered.Task.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); // Times out on the gate (1 s) and tears down ungated. From 44ea6595e515123fb855d3bd0acfaafb88c591cb Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:17:48 -0400 Subject: [PATCH 17/20] feat(arrays): advertise AbCip/AbLegacy/TwinCAT array writes in DriverArrayWriteSupport The Wave-2 lanes gave AbCip, AbLegacy and TwinCAT real whole-array write encoders (review 07 U10/U9), but the capability table still listed all three as scalar-only, so every array node materialised read-only and a client write was BadNotWritable before it reached the driver. Rows added, each listing only what WriteAsync encodes end to end: - AbCip: SInt/USInt/Int/UInt/DInt/UDInt/Real/Dt/LInt/ULInt/LReal, declared length, not a safetyTag. BOOL/STRING/Structure arrays stay read-only. - AbLegacy: Int/AnalogInt on N/B/A, Long on L, Float on F (file-width match), plain : address, <= 256 elements; not /bit, .sub, indirect, S:/I:/O:, nor native PLC-5. - TwinCAT: every codec element type, declared length; not a BOOL bit-in-word path. Elements convert in all three encoders, so the rows never key on an exact node DataType (ScadaBridge int[] -> Int16 stays writable); they leave out only node types whose elements never convert (DateTime/String over numeric, Boolean on AbLegacy, STRING under DateTime on TwinCAT: culture-dependent). Tests: driver-level parity for each driver (every element type x node type plus shape cases, against the real write path), the AbLegacy parity tests un-skipped, Core.Abstractions row tests, a composer test (three drivers writable, refused shapes read-only), and the existing AbCip negative cases moved to a BOOL array. Docs: Uns.md table, per-driver docs, data-plane rules. Refs #737 #738 #739 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .claude/rules/data-plane-auth.md | 4 +- docs/Uns.md | 21 +- docs/drivers/AbCip.md | 21 +- docs/drivers/AbLegacy.md | 20 +- docs/drivers/TwinCAT.md | 14 ++ .../DriverArrayWriteSupport.cs | 202 +++++++++++++++++- .../DriverArrayWriteSupportTests.cs | 108 +++++++++- .../AbCipArrayWriteSupportParityTests.cs | 133 ++++++++++++ .../AbLegacyArrayWriteSupportParityTests.cs | 167 +++++++++++++-- .../TwinCATArrayWriteSupportParityTests.cs | 135 ++++++++++++ .../ArrayWriteWireTests.cs | 10 +- .../AddressSpaceComposerDualNamespaceTests.cs | 75 ++++++- 12 files changed, 864 insertions(+), 46 deletions(-) create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipArrayWriteSupportParityTests.cs create mode 100644 tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayWriteSupportParityTests.cs diff --git a/.claude/rules/data-plane-auth.md b/.claude/rules/data-plane-auth.md index 44492c76..ca9f7886 100644 --- a/.claude/rules/data-plane-auth.md +++ b/.claude/rules/data-plane-auth.md @@ -151,7 +151,9 @@ Two things to know before relying on any of this: array node is writable only when the driver can write the array (`DriverArrayWriteSupport`: Modbus bounded by `maxRegistersPerWrite`/coil cap, S7 non-Bool/non-wide types, OpcUaClient, Galaxy signed types **with a declared `arrayLength`** — an undeclared length stays read-only because a short whole-array write would - resize the Galaxy attribute); `arrayLength >= 1` is required, and the node's `Writable && (!IsArray || + resize the Galaxy attribute; AbCip atomic non-BOOL types, AbLegacy Int/Long/Float on the matching data file + (no `/bit`/`.sub`/indirect/`S:`/`I:`/`O:`, not native PLC-5, ≤ 256), TwinCAT every codec type but a BOOL + bit-in-word path — #737/#738/#739; those three CONVERT elements, so they never key on an exact node type); `arrayLength >= 1` is required, and the node's `Writable && (!IsArray || ArrayWritable)` is projected onto the raw node and every UNS mirror. - **Anonymous is a normal grantable principal**, not a bypass. `OpcUa:AllowAnonymous` (default `true`, preserving today's behaviour) controls *admission*; when admitted the session authenticates as the diff --git a/docs/Uns.md b/docs/Uns.md index 8f59101e..8de7c783 100644 --- a/docs/Uns.md +++ b/docs/Uns.md @@ -327,12 +327,23 @@ carried on the plan records (`RawTagPlan.ArrayWritable`, mirrored onto every UNS | **S7** | Yes | `Byte`/`Int16`/`UInt16`/`Int32`/`UInt32`/`Float32` (one contiguous block). Not `Bool` (would clobber neighbouring bits) nor the wide / String / DateTime types | | **OpcUaClient** | Yes | All — forwarded upstream unchanged; the upstream server decides (shape included) | | **Galaxy** | Yes | OPC UA `Boolean`/`Int32`/`Int64`/`Float`/`Double`/`String`/`DateTime` (the `MxValueEncoder` array variants) — **only with a declared `arrayLength`**: an undeclared length materialises an unbounded node, and a short whole-array write would *resize* the Galaxy attribute (mxaccessgw contract), so such a node stays read-only | -| AB CIP, AB Legacy, TwinCAT, FOCAS, MQTT, MTConnect, Sql, Calculation | **No** — scalar-only write encoders; array nodes stay read-only and a client write is `BadNotWritable` | — | +| **AB CIP** | Yes (#737) | `SInt`/`USInt`/`Int`/`UInt`/`DInt`/`UDInt`/`Real`/`Dt`/`LInt`/`ULInt`/`LReal` (`AbCipArrayCodec`, both transports). Not `Bool` (bit-packed DWORDs), `String` or `Structure`, nor a `safetyTag` | +| **AB Legacy** | Yes (#738) | `Int`/`AnalogInt` on an N/B/A file, `Long` on an L file, `Float` on an F file — the encode width is the data file's, so a type that does not match its file stays read-only — on a plain `:` address, ≤ 256 elements. Not Bit / String / Timer / Counter / Control arrays, `/bit` or `.sub` addresses, indirect addresses, the `S:`/`I:`/`O:` files, nor any PLC-5 on the native transport | +| **TwinCAT** | Yes (#739) | Every element type `TwinCATArrayCodec` lays out (numeric, `Bool`, `String`/`WString`, temporal); multi-dimensional PLC arrays are written row-major from the flat value. Not a BOOL bit-in-word path (`MAIN.Word.3`) | +| FOCAS, MQTT, MTConnect, Sql, Calculation | **No** — scalar-only write encoders; array nodes stay read-only and a client write is `BadNotWritable` | — | -Modbus, S7 and Galaxy additionally need a **declared length** — `isArray: true` *and* an integer -`arrayLength >= 1`. The Modbus and S7 tag factories read anything else as a **scalar** tag (whose -scalar encoder cannot take an array), even though the composer still materialises an array node; -that node stays read-only. +Every "Yes" driver but OpcUaClient additionally needs a **declared length** — `isArray: true` *and* an +integer `arrayLength >= 1`. The tag factories read anything else as a **scalar** tag (whose scalar +encoder cannot take an array), even though the composer still materialises an array node; that node +stays read-only. + +**Elements convert; they are not exact-matched.** ScadaBridge writes CLR `int[]` / `List` to Int16 +nodes: the node manager converts a numeric array to the node's element type, and the AB CIP, AB Legacy +and TwinCAT encoders convert each element again to the controller's type (an element that does not fit +is `BadOutOfRange`). Those rows therefore do not key on an exact node `DataType`; they leave out only +node types whose elements never convert — a `DateTime` or `String` node over a numeric element, and a +`Boolean` node on AB Legacy. An array tag deployed read-only before its driver gained an encoder turns +writable on the next deploy (a surgical `ArrayWritable` update, no rebuild). Each "Yes" row is pinned against the driver's real encoder by a parity test in that driver's test project (`*ArrayWriteSupportParityTests`), so a driver gaining or losing an array encoder diff --git a/docs/drivers/AbCip.md b/docs/drivers/AbCip.md index 889e161f..4ddf1daa 100644 --- a/docs/drivers/AbCip.md +++ b/docs/drivers/AbCip.md @@ -252,7 +252,7 @@ The driver has two interchangeable wire transports, selected by one `DriverConfi write, read back, restored byte-exact). A user-approved **native array-element write** followed: `Program:EM01_PowerUnit._SEQ_EP[31]` 0→4242→0 (the array's only alias targets `[1]`, no routine indexes or block-copies it; all 32 elements snapshotted twice before, after the write and after the restore — only `[31]` changed). Still ab_server-only: whole-array - writes (the driver has none), libplctag member writes, and fragmented writes. #566 is answered (the controller drops TCP; see "A target-closed CIP connection"). Mixed STRING/array/scalar + writes (#737; see [1-D array support](#1-d-array-support)), libplctag member writes, and fragmented writes. #566 is answered (the controller drops TCP; see "A target-closed CIP connection"). Mixed STRING/array/scalar Multiple Service Packet reads (#582) are proven on the press (one 5-read MSP, values equal to single reads). Results: [native-ab-transport-gate-results](../plans/2026-10-07-native-ab-transport-gate-results.md). - **libplctag cannot be deleted yet.** Scalar, UDT-member and STRING native writes are proven on a CompactLogix @@ -262,7 +262,7 @@ The driver has two interchangeable wire transports, selected by one `DriverConfi libplctag member-write path is `ab_server`-only. See [Member writes](#member-writes). - ⚠️ RESIDUAL (#567): wide-struct **fragmented writes** (8-byte alignment) are unit-tested only. A write fragments only above `MaxRequestBytes` (4000 B on a Large Forward Open), so it needs a ~4 KB value; Z-35162 has no safe one (a DINT[32] - or a STRING is a single Write Tag, and AbCip has no whole-array write anyway). It stays open until a bench controller + or a STRING is a single Write Tag — and a whole DINT[32] array write, #737, is 128 B). It stays open until a bench controller or an approved large scratch tag exists. - Native browse (#572, resolved in code): listing paging no longer assumes ascending instance order, pages are paced, and a browse's template reads share one lease. Observed on Z-35162 (2026-10-09, 41 @@ -361,7 +361,17 @@ integration fixture is down during normal dev (Docker host fixture currently off **Live-verify** — integration-fixture-gated (requires `ab_server` Docker fixture on `10.100.0.35:44818`). -**Deferrals** — array *writes*, multi-dimensional arrays, per-element historization. Safety-partition array tags (`SafetyTag: true`) are excluded (GuardLogix write constraints extend to array writes). +**Array writes (#737, review 07 U10)** — a whole-array write of exactly `arrayLength` elements, encoded by +`AbCipArrayCodec` little-endian at the element width on both transports. Element values **convert** +(ScadaBridge writes `int[]` to INT arrays); a value that does not fit is `BadOutOfRange`, one that cannot convert +`BadTypeMismatch`, a different length `BadOutOfRange`, all before anything is sent. The controller buffer must be +width × count (else `BadTypeMismatch`, checked before sending). Writable element types: `SInt`/`USInt`/`Int`/`UInt`/ +`DInt`/`UDInt`/`Real`/`Dt`/`LInt`/`ULInt`/`LReal`. **BOOL arrays** (bit-packed DWORDs), **STRING** and **structure** +arrays are refused `BadNotSupported`, so their nodes stay read-only (`DriverArrayWriteSupport`); so does a +safety-partition array tag (`safetyTag: true`, `BadNotWritable`). An array deployed read-only before #737 turns +writable on the next deploy (a surgical `ArrayWritable` update). Proven on `ab_server` only until the hardware gate runs. + +**Deferrals** — multi-dimensional arrays, per-element historization, whole-array writes on hardware. See [Uns.md §Array tags](../Uns.md#array-tags-1-d) for the cross-driver coverage matrix and the UI authoring flow. @@ -454,8 +464,9 @@ when it is browse-committed in `/raw`, and that tag writes like any other atomic `writable` key. The node is writable only once the operator sets the raw tag to `ReadWrite` (the node-manager gate); an authored `"writable": false` still refuses at the driver (`BadNotWritable`). The browse's `SecurityClass` (members inherit the parent symbol's read-only bit) is not consulted by the - commit. An element of a member array is authored by hand as `Seg.Points[2]`; a whole member array is - not writable (AbCip has no array encoder, `DriverArrayWriteSupport`). + commit. An element of a member array is authored by hand as `Seg.Points[2]`; a whole member array of an + atomic type is writable as an array tag (`isArray` + `arrayLength`, #737), a BOOL / STRING / structure member + array is not (`DriverArrayWriteSupport`). - **The request names the member.** One CIP symbolic segment per member (plus an element segment), the member's own CIP type, the member's own bytes — never the parent UDT, never a hidden host. Pinned at the request-byte level by `NativeAbCipTagRuntimeTests.A_member_path_write_names_the_member_and_sends_only_its_own_bytes` diff --git a/docs/drivers/AbLegacy.md b/docs/drivers/AbLegacy.md index 88ae97a6..d2774cc0 100644 --- a/docs/drivers/AbLegacy.md +++ b/docs/drivers/AbLegacy.md @@ -304,8 +304,18 @@ N values of the declared `AbLegacyDataType`. Maximum element count is 256 (PCCC limit). This closes the previously-noted limitation ("single-element addressing today") for the -array read path. The opt-in native transport also writes multi-element arrays (chunked); on libplctag -array *writes* remain a follow-up. +array read path. + +**Array writes (#738, review 07 U10 / r0 I3)** — on both transports, a whole-array write encodes exactly +`arrayLength` elements at the **data file's** element width (N / B / A words, L longs, F floats). Element values +**convert** (ScadaBridge writes `int[]` to Int16 nodes); a length mismatch or an element that overflows is +`BadOutOfRange`, a non-numeric value `BadTypeMismatch`, before any runtime exists. Writable: `Int` / `AnalogInt` on +an N / B / A file, `Long` on an L file, `Float` on an F file, on a plain `:` address of at most +256 elements. These nodes stay **read-only** (`DriverArrayWriteSupport`): Bit arrays (bit-packed words), String / +Timer / Counter / Control arrays (rejected at `FromTagConfig`), `/bit` and `.sub` addresses, a type that does not +match its file (`BadTypeMismatch`), any PLC-5 on the native transport (writes refused before the wire), the +`S:` / `I:` / `O:` files (libplctag refuses their translated names), and — deliberately — indirect addresses, whose +element moves with the index word. An array deployed read-only before #738 turns writable on the next deploy. **Unit test coverage** — `tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/` covers the multi-element read path via the fake tag runtime. The AB Legacy integration fixture is @@ -313,8 +323,10 @@ down during normal dev. **Live-verify** — integration-fixture-gated (requires the AB Legacy Docker fixture). -**Deferrals** — array *writes* on libplctag (the opt-in native transport writes them), multi-dimensional arrays, per-element historization. -`arrayLength > 256` is rejected at init with an explicit error (PCCC payload cap). +**Deferrals** — multi-dimensional arrays, per-element historization, array writes on hardware +(`AbLegacyArrayWriteIntegrationTests` is green on `ab_server` and ready for a real controller via +`AB_LEGACY_ARRAY_WRITE_BASE`). +`arrayLength > 256` is clamped to 256 by the tag factory (PCCC payload cap), so such a node stays read-only. See [Uns.md §Array tags](../Uns.md#array-tags-1-d) for the cross-driver coverage matrix and the UI authoring flow. diff --git a/docs/drivers/TwinCAT.md b/docs/drivers/TwinCAT.md index 22477d55..645fd025 100644 --- a/docs/drivers/TwinCAT.md +++ b/docs/drivers/TwinCAT.md @@ -362,6 +362,20 @@ element-typed CLR array (`bool[]`, `sbyte[]`, `byte[]`, `short[]`, `ushort[]`, ` down (found live, below). A bit outside the parent's width, or a bit of a non-integer parent, is refused with `BadTypeMismatch`. +### Array writes (#739, review 07 U9) + +An array node is **writable** (`DriverArrayWriteSupport`) for every element type the codec lays out — +numeric, `Bool` (one byte per element, as TwinCAT stores a BOOL array), `STRING`/`WSTRING` and the temporal +types. The write takes exactly `arrayLength` elements and **converts** each one (`ConvertForWrite`; ScadaBridge +writes `int[]` to an Int16 node); a length mismatch or an out-of-range element is `BadOutOfRange` and a scalar +value `BadTypeMismatch`, before any wire traffic. `WriteArrayAsync` then plans the layout from the resolved PLC +array with the same `CheckArray` the read uses and encodes with `TwinCATArrayCodec.Encode` (the inverse of the +decode); a shorter `arrayLength` writes the leading sub-range, and a multi-dimensional PLC array is written +row-major from the flat 1-D value. A BOOL **bit-in-word** path (`MAIN.Word.3`) stays read-only as an array: it is a +scalar bit write. Under a `DateTime` node only `DATE`/`DT`/`WSTRING` elements are writable, and under a `String` +node only `STRING`/`WSTRING` (other conversions would depend on the text). An array deployed read-only before #739 +turns writable on the next deploy. Unit- and XAR-gate-tested; real-hardware proof is pending. + ### Notification size cap ADS documents no notification size limit, and a live TwinCAT 2.11 target registered and diff --git a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverArrayWriteSupport.cs b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverArrayWriteSupport.cs index f16f8c21..08b87af5 100644 --- a/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverArrayWriteSupport.cs +++ b/src/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions/DriverArrayWriteSupport.cs @@ -35,12 +35,39 @@ namespace ZB.MOM.WW.OtOpcUa.Core.Abstractions; /// DateTime — and only with a declared arrayLength: an undeclared length materialises an /// unbounded node, and a short whole-array write to it would RESIZE the Galaxy attribute (the /// mxaccessgw gateway.md contract), which is never what a client meant. -/// Everything else (AbCip, AbLegacy, TwinCAT, FOCAS, MQTT, MTConnect, Sql, Calculation) — scalar -/// write encoders only; array nodes stay read-only. +/// AbCip — dataType SInt / USInt / Int / UInt / DInt / UDInt / Real / Dt / LInt / ULInt / +/// LReal (AbCipArrayCodec, both transports), and not a safetyTag. BOOL arrays (bit-packed +/// DWORDs), STRING and structure arrays have no array encoder. +/// AbLegacy — dataType Int / AnalogInt on an N / B / A file, Long on an L file, Float +/// on an F file (the encode width is the DATA FILE's; a type that does not match its file is refused), +/// on a plain <file><number>:<word> address of at most 256 elements (the PCCC +/// file ceiling the factory clamps to). Refused and so left out: Bit arrays (bit-packed words), String / +/// Timer / Counter / Control arrays (structured; the factory rejects them), /bit and .sub +/// addresses, a PLC-5 device on the native transport (every PLC-5 write is refused before the wire), and +/// the S: / I: / O: files (libplctag refuses their translated names). Indirect +/// addresses (N7:[N7:0]) are kept read-only deliberately: the element moves with the index word +/// and the index read is not atomic with the write. +/// TwinCAT — every element type TwinCATArrayCodec lays out (all but Structure, which +/// the factory refuses), STRING / WSTRING included, except a BOOL bit-in-word path +/// (MAIN.Word.3), which is a scalar bit write. A multi-dimensional PLC array is written row-major +/// from the flat 1-D value. The PLC-side shape (element width, declared length ≤ the PLC's) is checked +/// against the resolved symbol at write time, which no compose-time table can know. +/// Everything else (FOCAS, MQTT, MTConnect, Sql, Calculation) — scalar write encoders only; array +/// nodes stay read-only. /// -/// Modbus, S7 and Galaxy require a declared length — isArray: true AND an integer -/// arrayLength >= 1. The Modbus and S7 tag factories treat anything else as a SCALAR tag (whose +/// Modbus, S7, Galaxy, AbCip, AbLegacy and TwinCAT require a declared length — isArray: true AND +/// an integer arrayLength >= 1. Their tag factories treat anything else as a SCALAR tag (whose /// scalar encoder cannot take an array value), though the composer still materialises an array node. +/// Elements convert; they are not exact-matched. ScadaBridge writes CLR arrays such as +/// int[] / List<int> to Int16 nodes; the node manager converts a numeric array to the +/// node's element type, and the AbCip, AbLegacy and TwinCAT encoders then Convert each element to the +/// controller's type (an element that does not fit is BadOutOfRange). So those rows do not key on an +/// exact node DataType; they leave out only the node types whose elements NEVER convert: a +/// DateTime or String node over a numeric element (AbCip, AbLegacy, TwinCAT — TwinCAT keeps +/// DATE / DT and WSTRING under a DateTime node, and STRING / WSTRING under a String node; a DateTime's text +/// form is culture-dependent, so a STRING under a DateTime node is left out), and a Boolean node on +/// AbLegacy, whose encoder refuses bool elements. An unknown DataType name (served as BaseDataType) hands the +/// driver whatever the client sent and is not refused. /// Multi-dimensional arrays (ValueRank > 1) are out of scope altogether — every array node /// is materialised 1-D (ValueRank = OneDimension), so nothing here considers more than one rank. /// @@ -72,6 +99,34 @@ public static class DriverArrayWriteSupport "Boolean", "Int32", "Int64", "Float", "Double", "String", "DateTime", }; + // AbCipArrayCodec.ElementWidth: the atomic element types with a fixed wire width. + private static readonly HashSet AbCipArrayTypes = new(StringComparer.OrdinalIgnoreCase) + { + "SInt", "USInt", "Int", "UInt", "DInt", "UDInt", "Real", "Dt", "LInt", "ULInt", "LReal", + }; + + /// The PCCC data-file element count ceiling (AbLegacyArray.MaxElements): the AbLegacy tag + /// factory CLAMPS a longer arrayLength to it, so a node of the authored (longer) length never matches. + public const int AbLegacyMaxArrayElements = 256; + + // AbLegacyTagDefinitionFactory.ArrayFileElementType for the files the table admits, keyed by file letter, to the + // authored element types that match that file's width (ArrayFileMismatch). I / O / S are deliberately absent: + // libplctag refuses a write to their translated names, and the table does not widen output-image / status-file + // writes on the native transport alone. + private static readonly Dictionary AbLegacyFileElementTypes = new(StringComparer.OrdinalIgnoreCase) + { + ["N"] = ["Int", "AnalogInt"], ["B"] = ["Int", "AnalogInt"], ["A"] = ["Int", "AnalogInt"], + ["L"] = ["Long"], + ["F"] = ["Float"], + }; + + // TwinCATArrayCodec.Encode: every TwinCATDataType but Structure (which the factory refuses outright). + private static readonly HashSet TwinCATArrayTypes = new(StringComparer.OrdinalIgnoreCase) + { + "Bool", "SInt", "USInt", "Int", "UInt", "DInt", "UDInt", "LInt", "ULInt", "Real", "LReal", + "String", "WString", "Time", "Date", "DateTime", "TimeOfDay", + }; + /// /// Whether a driver of can write a 1-D array tag of the given shape. /// @@ -119,11 +174,150 @@ public static class DriverArrayWriteSupport var native = ReadEnumName(root, "dataType") ?? "Int16"; return S7ArrayTypes.Contains(native); } + case DriverTypeNames.AbCip: + return AbCipSupports(dataType, tagConfig); + case DriverTypeNames.AbLegacy: + return AbLegacySupports(dataType, tagConfig, driverConfigs); + case DriverTypeNames.TwinCAT: + return TwinCATSupports(dataType, tagConfig); default: return false; } } + // AbCipTagDefinitionFactory + AbCipDriver.WriteAsync: a non-blank string tagPath, an atomic element type the codec + // has a width for (BOOL arrays are bit-packed DWORDs, STRING / structure elements are structures), a declared + // length, and not a safety tag (refused BadNotWritable). Each element is CONVERTED, so any numeric / Boolean node + // type works; DateTime and (non-numeric) String elements never convert to a Logix integer or REAL. + private static bool AbCipSupports(string? dataType, string? tagConfig) + { + if (!TryParseObject(tagConfig, out var root) || !HasNonBlankString(root, "tagPath") + || ReadArrayLength(root) is null + || (root.TryGetProperty("safetyTag", out var safety) && safety.ValueKind == JsonValueKind.True)) + return false; + var native = ReadEnumName(root, "dataType")?.Trim() ?? "DInt"; + return AbCipArrayTypes.Contains(native) && dataType is not ("String" or "DateTime"); + } + + // AbLegacyTagDefinitionFactory + AbLegacyDriver.WriteAsync: a flat Int / AnalogInt / Long / Float array on a plain + // `:` address whose data file's element width matches (ArrayFileMismatch); no /bit, no .sub, + // no indirect [..] address; at most 256 elements (the factory clamps a longer length, so the node never matches); + // and not a PLC-5 on the native transport (refused before the wire). Elements CONVERT from any numeric node type; + // bool, string and DateTime elements are refused. + private static bool AbLegacySupports(string? dataType, string? tagConfig, string?[]? driverConfigs) + { + if (!TryParseObject(tagConfig, out var root) + || !root.TryGetProperty("address", out var addressEl) || addressEl.ValueKind != JsonValueKind.String + || ReadArrayLength(root) is not { } count || count > AbLegacyMaxArrayElements + || dataType is "Boolean" or "String" or "DateTime") + return false; + var native = ReadEnumName(root, "dataType")?.Trim() ?? "Int"; + if (!TryParsePlainPcccAddress(addressEl.GetString(), out var fileLetter) + || !AbLegacyFileElementTypes.TryGetValue(fileLetter, out var matching) + || !matching.Contains(native, StringComparer.OrdinalIgnoreCase)) + return false; + return !AbLegacyIsNativePlc5(driverConfigs); + } + + // `:` and nothing else (trimmed) — what AbLegacyAddress.TryParse yields with no bit + // index, no sub-element and no indirection, for the file letters that carry a file number. + private static bool TryParsePlainPcccAddress(string? address, out string fileLetter) + { + fileLetter = ""; + if (string.IsNullOrWhiteSpace(address)) return false; + var src = address.Trim(); + var colon = src.IndexOf(':'); + if (colon <= 0) return false; + var letterEnd = 0; + while (letterEnd < colon && char.IsAsciiLetter(src[letterEnd])) letterEnd++; + if (letterEnd == 0 || letterEnd == colon) return false; // no letter, or no file number + var fileNumber = src[letterEnd..colon]; + var word = src[(colon + 1)..]; + if (!fileNumber.All(char.IsAsciiDigit) || !int.TryParse(fileNumber, out _) + || word.Length == 0 || !word.All(char.IsAsciiDigit) || !int.TryParse(word, out _)) + return false; + fileLetter = src[..letterEnd]; + return true; + } + + // Conservative: ANY config naming the native transport AND ANY config naming a PLC-5 (a root `plcFamily`, as a + // DeviceConfig carries it, or one in a `devices` array, as a DriverConfig carries it). Keys and values are matched + // case-insensitively, like the driver's DTO binding and enum parse. An absent transport is libplctag for every + // family (AbLegacyTransportResolver), which writes PLC-5. + private static bool AbLegacyIsNativePlc5(string?[]? driverConfigs) + { + var native = false; + var plc5 = false; + foreach (var config in driverConfigs ?? []) + { + if (!TryParseObject(config, out var root)) continue; + foreach (var prop in root.EnumerateObject()) + { + if (IsNamed(prop, "transport") && IsStringValue(prop.Value, "Native")) native = true; + else if (IsNamed(prop, "plcFamily") && IsStringValue(prop.Value, "Plc5")) plc5 = true; + else if (IsNamed(prop, "devices") && prop.Value.ValueKind == JsonValueKind.Array) + { + foreach (var device in prop.Value.EnumerateArray()) + { + if (device.ValueKind != JsonValueKind.Object) continue; + foreach (var dp in device.EnumerateObject()) + if (IsNamed(dp, "plcFamily") && IsStringValue(dp.Value, "Plc5")) plc5 = true; + } + } + } + } + return native && plc5; + } + + // TwinCATTagDefinitionFactory + TwinCATDriver.WriteAsync: a non-blank string symbolPath, a declared length, any + // element type the codec lays out (not Structure, which the factory refuses), and not a BOOL bit-in-word path + // (`MAIN.Word.3`: a scalar bit write, which an array value cannot feed). Elements CONVERT. A String node only feeds + // a STRING / WSTRING element (anything else would depend on the text happening to parse). A DateTime node only + // feeds a DATE / DT element or a WSTRING: its text form is CULTURE-dependent and may carry characters a STRING's + // strict Windows-1252 encoding refuses (ICU's en-US "12:00:00 PM"). + private static bool TwinCATSupports(string? dataType, string? tagConfig) + { + if (!TryParseObject(tagConfig, out var root) || !HasNonBlankString(root, "symbolPath") + || ReadArrayLength(root) is null) + return false; + var native = ReadEnumName(root, "dataType")?.Trim() ?? "DInt"; + if (!TwinCATArrayTypes.Contains(native)) return false; + var isWString = native.Equals("WString", StringComparison.OrdinalIgnoreCase); + var isString = isWString || native.Equals("String", StringComparison.OrdinalIgnoreCase); + var isDate = native.Equals("Date", StringComparison.OrdinalIgnoreCase) || native.Equals("DateTime", StringComparison.OrdinalIgnoreCase); + if (dataType == "String" && !isString) return false; + if (dataType == "DateTime" && !isWString && !isDate) return false; + return !(native.Equals("Bool", StringComparison.OrdinalIgnoreCase) + && HasTwinCATBitSuffix(root.GetProperty("symbolPath").GetString()!)); + } + + // TwinCATSymbolPath.TryParse's bit-index rule: a last top-level `.` segment of 0..31 with no `[`, behind at least + // one other segment (a TwinCAT 2 global's leading dot belongs to its root segment). + private static bool HasTwinCATBitSuffix(string symbolPath) + { + var src = symbolPath.Trim(); + if (src.StartsWith('.')) src = src[1..]; + var depth = 0; + var lastDot = -1; + for (var i = 0; i < src.Length; i++) + { + if (src[i] == '[') depth++; + else if (src[i] == ']') depth--; + else if (src[i] == '.' && depth == 0) lastDot = i; + } + if (lastDot < 0) return false; + var last = src[(lastDot + 1)..]; + return int.TryParse(last, out var bit) && bit is >= 0 and <= 31 && !last.Contains('['); + } + + private static bool HasNonBlankString(JsonElement root, string name) => + root.TryGetProperty(name, out var el) && el.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(el.GetString()); + + private static bool IsNamed(JsonProperty prop, string name) => string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase); + + private static bool IsStringValue(JsonElement el, string value) => + el.ValueKind == JsonValueKind.String && string.Equals(el.GetString()?.Trim(), value, StringComparison.OrdinalIgnoreCase); + /// /// Whether a driver of is known to REFUSE an /// array value written to one of its SCALAR tags, so the node manager may refuse such a write inline diff --git a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/DriverArrayWriteSupportTests.cs b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/DriverArrayWriteSupportTests.cs index 2da30f96..18137306 100644 --- a/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/DriverArrayWriteSupportTests.cs +++ b/tests/Core/ZB.MOM.WW.OtOpcUa.Core.Abstractions.Tests/DriverArrayWriteSupportTests.cs @@ -138,11 +138,117 @@ public sealed class DriverArrayWriteSupportTests public void OpcUaClient_forwards_every_array() => DriverArrayWriteSupport.Supports(DriverTypeNames.OpcUaClient, "UInt16", "{\"nodeId\":\"ns=2;s=x\"}").ShouldBeTrue(); + // ---- AbCip / AbLegacy / TwinCAT (review 07 U10 / U9: their array encoders landed in #737 / #738 / #739) ---- + + /// AbCip: the atomic element types AbCipArrayCodec encodes. BOOL (bit-packed DWORDs), STRING and + /// structure arrays stay read-only, as do a safety tag, an undeclared length and a node type no element converts + /// from. + [Theory] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInt\"" + Arr2 + "}", "Int32", true)] + [InlineData("{\"tagPath\":\"Arr\"" + Arr2 + "}", "Int32", true)] // dataType ⇒ DInt + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"int\"" + Arr2 + "}", "Int16", true)] // case-insensitive, like the factory + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"LReal\"" + Arr2 + "}", "Double", true)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"Dt\"" + Arr2 + "}", "Int32", true)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"Bool\"" + Arr2 + "}", "Boolean", false)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"String\"" + Arr2 + "}", "String", false)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"Structure\"" + Arr2 + "}", "Int32", false)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInnt\"" + Arr2 + "}", "Int32", false)] // typo ⇒ the factory refuses the tag + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInt\",\"safetyTag\":true" + Arr2 + "}", "Int32", false)] + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInt\",\"isArray\":true}", "Int32", false)] // no length ⇒ scalar + [InlineData("{\"dataType\":\"DInt\"" + Arr2 + "}", "Int32", false)] // no tagPath ⇒ not an AbCip tag + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInt\"" + Arr2 + "}", "DateTime", false)] // DateTime elements never convert + [InlineData("{\"tagPath\":\"Arr\",\"dataType\":\"DInt\"" + Arr2 + "}", "String", false)] // text converts only when numeric + public void AbCip_rows_follow_the_element_type(string tagConfig, string dataType, bool expected) => + DriverArrayWriteSupport.Supports(DriverTypeNames.AbCip, dataType, tagConfig).ShouldBe(expected); + + /// AbLegacy: Int / AnalogInt / Long / Float on a plain address of the MATCHING data file. Every shape the + /// write path refuses stays read-only. + [Theory] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", true)] + [InlineData("{\"address\":\"N7:0\"" + Arr2 + "}", "Int16", true)] // dataType ⇒ Int + [InlineData("{\"address\":\" n7:10 \",\"dataType\":\"analogint\"" + Arr2 + "}", "Int16", true)] // trimmed, case-insensitive + [InlineData("{\"address\":\"B3:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", true)] + [InlineData("{\"address\":\"A9:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", true)] + [InlineData("{\"address\":\"L9:0\",\"dataType\":\"Long\"" + Arr2 + "}", "Int32", true)] + [InlineData("{\"address\":\"F8:0\",\"dataType\":\"Float\"" + Arr2 + "}", "Float", true)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":256}", "Int16", true)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":257}", "Int16", false)] // the driver clamps to 256 + [InlineData("{\"address\":\"B3:0\",\"dataType\":\"Bit\"" + Arr2 + "}", "Boolean", false)] + [InlineData("{\"address\":\"ST9:0\",\"dataType\":\"String\"" + Arr2 + "}", "String", false)] + [InlineData("{\"address\":\"T4:0\",\"dataType\":\"TimerElement\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"N7:0/3\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"T4:0.ACC\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"N7:[N7:0]\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"F8:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] // type / file mismatch + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Long\"" + Arr2 + "}", "Int32", false)] + [InlineData("{\"address\":\"S:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] // libplctag refuses a translated name + [InlineData("{\"address\":\"O:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\",\"isArray\":true}", "Int16", false)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Innt\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}", "Boolean", false)] // bool elements are refused + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}", "String", false)] + [InlineData("{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}", "DateTime", false)] + public void AbLegacy_rows_follow_the_element_type_and_data_file(string tagConfig, string dataType, bool expected) => + DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, dataType, tagConfig).ShouldBe(expected); + + /// The native transport refuses every PLC-5 write before the wire, so a native PLC-5 device keeps its + /// arrays read-only, wherever the transport and the family are configured. libplctag PLC-5 is the control. + [Theory] + [InlineData("{\"transport\":\"Native\",\"devices\":[{\"hostAddress\":\"ab://h/1,0\",\"plcFamily\":\"Plc5\"}]}", "{}", false)] + [InlineData("{\"Transport\":\"native\"}", "{\"PlcFamily\":\"plc5\"}", false)] + [InlineData("{\"transport\":\"Native\"}", "{\"plcFamily\":\"Slc500\"}", true)] + [InlineData("{\"transport\":\"Libplctag\"}", "{\"plcFamily\":\"Plc5\"}", true)] + [InlineData("{}", "{\"plcFamily\":\"Plc5\"}", true)] // absent ⇒ libplctag + public void AbLegacy_native_PLC5_stays_read_only(string driverConfig, string deviceConfig, bool expected) => + DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, "Int16", + "{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}", driverConfig, deviceConfig).ShouldBe(expected); + + /// TwinCAT: every element type TwinCATArrayCodec encodes (the factory refuses Structure tags). A + /// BOOL bit-in-word path writes a scalar bit and is refused for an array value. + [Theory] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\"" + Arr2 + "}", "Int32", true)] // dataType ⇒ DInt + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Bool\"" + Arr2 + "}", "Boolean", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"String\"" + Arr2 + "}", "String", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"WString\"" + Arr2 + "}", "String", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Time\"" + Arr2 + "}", "Double", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"DateTime\"" + Arr2 + "}", "DateTime", true)] + [InlineData("{\"symbolPath\":\"MAIN.Grid[0,0]\",\"dataType\":\"Int\"" + Arr2 + "}", "Int16", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Structure\"" + Arr2 + "}", "Int32", false)] + [InlineData("{\"symbolPath\":\"MAIN.Word.3\",\"dataType\":\"Bool\"" + Arr2 + "}", "Boolean", false)] // bit-in-word + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Int\",\"isArray\":true}", "Int16", false)] + [InlineData("{\"dataType\":\"Int\"" + Arr2 + "}", "Int16", false)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Int\"" + Arr2 + "}", "DateTime", false)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Time\"" + Arr2 + "}", "DateTime", false)] // a duration, not a date + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Int\"" + Arr2 + "}", "String", false)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"WString\"" + Arr2 + "}", "DateTime", true)] + [InlineData("{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"String\"" + Arr2 + "}", "DateTime", false)] // culture-dependent text vs strict Windows-1252 + public void TwinCAT_rows_follow_the_element_type(string tagConfig, string dataType, bool expected) => + DriverArrayWriteSupport.Supports(DriverTypeNames.TwinCAT, dataType, tagConfig).ShouldBe(expected); + + /// ScadaBridge writes CLR int[] / List<int> to Int16 nodes, and an unknown DataType + /// name (served as BaseDataType) hands the driver whatever the client sent. The element CONVERTS in all three + /// encoders, so neither case may make the row read-only. + [Theory] + [InlineData(DriverTypeNames.AbCip, "{\"tagPath\":\"Arr\",\"dataType\":\"Int\"" + Arr2 + "}")] + [InlineData(DriverTypeNames.AbLegacy, "{\"address\":\"N7:0\",\"dataType\":\"Int\"" + Arr2 + "}")] + [InlineData(DriverTypeNames.TwinCAT, "{\"symbolPath\":\"MAIN.Arr\",\"dataType\":\"Int\"" + Arr2 + "}")] + public void Converting_drivers_do_not_key_on_an_exact_node_type(string driverType, string tagConfig) + { + foreach (var dataType in new[] { "Int16", "Int32", "Int64", "UInt16", "Double", "Float32", "Reference" }) + DriverArrayWriteSupport.Supports(driverType, dataType, tagConfig).ShouldBeTrue($"{driverType} {dataType}"); + DriverArrayWriteSupport.Supports(driverType, null, tagConfig).ShouldBeTrue($"{driverType} "); + } + /// Every driver type NOT in the table — and an unknown or absent one — stays read-only. [Fact] public void Every_other_driver_type_is_not_array_writable() { - var listed = new[] { DriverTypeNames.Modbus, DriverTypeNames.S7, DriverTypeNames.OpcUaClient, DriverTypeNames.Galaxy }; + var listed = new[] + { + DriverTypeNames.Modbus, DriverTypeNames.S7, DriverTypeNames.OpcUaClient, DriverTypeNames.Galaxy, + DriverTypeNames.AbCip, DriverTypeNames.AbLegacy, DriverTypeNames.TwinCAT, + }; var others = DriverTypeNames.All.Except(listed).ToList(); others.ShouldNotBeEmpty(); // A blob every listed driver would accept, so only the driver type can make the answer false. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipArrayWriteSupportParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipArrayWriteSupportParityTests.cs new file mode 100644 index 00000000..488020db --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests/AbCipArrayWriteSupportParityTests.cs @@ -0,0 +1,133 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.AbCip.Tests; + +/// +/// Pins 's AbCip row against the REAL write path (#737, review 07 U10), the +/// same way the Modbus, S7 and Galaxy parity tests do: the server advertises an array node writable exactly when +/// encodes a whole-array value for it (through the production +/// , which the fake runtime's EncodeArray calls), so a driver change that adds +/// or drops an array element type fails here until the table follows. +/// +/// +/// The value written is the CLR array the server hands the driver for the node's DataType: the node manager +/// converts a numeric client array (ScadaBridge's int[]) to the node's element type first, and the codec +/// then converts each element to the controller's type. The fake controller holds exactly +/// width × count bytes, so only the element type and the value can make the write fail. +/// +[Trait("Category", "Unit")] +public sealed class AbCipArrayWriteSupportParityTests +{ + private const string Device = "ab://10.0.0.5/1,0"; + + private static Array ClrArray(string dataType, int n) => dataType switch + { + "Boolean" => Enumerable.Repeat(true, n).ToArray(), + "SByte" => Enumerable.Repeat((sbyte)1, n).ToArray(), + "Byte" => Enumerable.Repeat((byte)1, n).ToArray(), + "Int16" => Enumerable.Repeat((short)1, n).ToArray(), + "UInt16" => Enumerable.Repeat((ushort)1, n).ToArray(), + "Int32" => Enumerable.Repeat(1, n).ToArray(), + "UInt32" => Enumerable.Repeat(1u, n).ToArray(), + "Int64" => Enumerable.Repeat(1L, n).ToArray(), + "UInt64" => Enumerable.Repeat(1UL, n).ToArray(), + "Float" => Enumerable.Repeat(1f, n).ToArray(), + "Double" => Enumerable.Repeat(1d, n).ToArray(), + // Non-numeric text: a String node over a numeric element converts only when the text happens to be a + // number, which is not a write the server can promise — the table keeps it read-only. + "String" => Enumerable.Repeat("a", n).ToArray(), + "DateTime" => Enumerable.Repeat(DateTime.UtcNow, n).ToArray(), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + private static readonly string[] NodeTypes = + ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64", "Float", "Double", "String", "DateTime"]; + + private static async Task DriverEncodesAsync(string tagConfig, AbCipDataType elementType, string nodeType, int count) + { + // The controller holds exactly what an encodable element array of this length occupies. + var size = (AbCipArrayCodec.ElementWidth(elementType) ?? 1) * count; + var factory = new FakeAbCipTagFactory { Customise = p => new FakeAbCipTag(p) { Size = size } }; + using var drv = new AbCipDriver(new AbCipDriverOptions + { + Devices = [new AbCipDeviceOptions(Device)], + RawTags = [new RawTagEntry("Arr", tagConfig, WriteIdempotent: false, DeviceName: Device)], + Probe = new AbCipProbeOptions { Enabled = false }, + }, "abcip-parity", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var r = await drv.WriteAsync([new WriteRequest("Arr", ClrArray(nodeType, count))], CancellationToken.None); + + return r[0].StatusCode == AbCipStatusMapper.Good + && factory.Tags.TryGetValue("Arr", out var tag) + && tag.EncodedArrayCount == count && tag.WriteCount == 1; + } + + public static TheoryData TypeMatrix() + { + var data = new TheoryData(); + foreach (var t in Enum.GetValues()) + foreach (var n in NodeTypes) + data.Add(t, n); + return data; + } + + /// Every element type × every node DataType: the table answers exactly what the write path does. + [Theory] + [MemberData(nameof(TypeMatrix))] + public async Task Table_agrees_with_the_write_path_for_every_element_and_node_type(AbCipDataType dataType, string nodeType) + { + var json = $$"""{"tagPath":"Arr","dataType":"{{dataType}}","isArray":true,"arrayLength":3}"""; + + var driver = await DriverEncodesAsync(json, dataType, nodeType, 3); + + DriverArrayWriteSupport.Supports(DriverTypeNames.AbCip, nodeType, json).ShouldBe(driver, $"{dataType} on a {nodeType} node"); + } + + /// + /// Shapes beyond the element type, both directions: a safety tag (never writable), an undeclared length (a + /// scalar to the factory, whose scalar encoder cannot take an array), a 1-element array, an element-offset path, + /// and their positive controls. + /// + [Theory] + [InlineData("""{"tagPath":"Arr","dataType":"DInt","isArray":true,"arrayLength":3}""", 3)] + [InlineData("""{"tagPath":"Arr","dataType":"DInt","isArray":true,"arrayLength":1}""", 1)] + [InlineData("""{"tagPath":"Arr","isArray":true,"arrayLength":3}""", 3)] // dataType ⇒ DInt + [InlineData("""{"tagPath":"Arr","dataType":"DInt","safetyTag":true,"isArray":true,"arrayLength":3}""", 3)] + [InlineData("""{"tagPath":"Arr","dataType":"DInt","isArray":true}""", 3)] + [InlineData("""{"tagPath":"Arr","dataType":"DInt","isArray":true,"arrayLength":0}""", 3)] + [InlineData("""{"tagPath":"Arr","dataType":"DInt","isArray":false,"arrayLength":3}""", 3)] + public async Task Table_agrees_with_the_write_path_for_each_shape(string tagConfig, int nodeLength) + { + var driver = await DriverEncodesAsync(tagConfig, AbCipDataType.DInt, "Int32", nodeLength); + + DriverArrayWriteSupport.Supports(DriverTypeNames.AbCip, "Int32", tagConfig).ShouldBe(driver, tagConfig); + } + + /// The ScadaBridge wire shape reaches the codec: a CLR int[] (and List<int>) to an + /// INT array on an Int16 node the table advertises writable encodes. Positive control for the conversion rule. + [Fact] + public async Task A_ScadaBridge_int_list_to_an_INT_array_encodes() + { + const string json = """{"tagPath":"Arr","dataType":"Int","isArray":true,"arrayLength":3}"""; + DriverArrayWriteSupport.Supports(DriverTypeNames.AbCip, "Int16", json).ShouldBeTrue(); + + foreach (object value in new object[] { new[] { 1, -2, 300 }, new List { 1, -2, 300 } }) + { + var factory = new FakeAbCipTagFactory { Customise = p => new FakeAbCipTag(p) { Size = 6 } }; + using var drv = new AbCipDriver(new AbCipDriverOptions + { + Devices = [new AbCipDeviceOptions(Device)], + RawTags = [new RawTagEntry("Arr", json, WriteIdempotent: false, DeviceName: Device)], + Probe = new AbCipProbeOptions { Enabled = false }, + }, "abcip-parity-sb", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + (await drv.WriteAsync([new WriteRequest("Arr", value)], CancellationToken.None))[0].StatusCode + .ShouldBe(AbCipStatusMapper.Good); + factory.Tags["Arr"].EncodedArrayBytes.ShouldBe(Convert.FromHexString("0100FEFF2C01")); + } + } +} diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/AbLegacyArrayWriteSupportParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/AbLegacyArrayWriteSupportParityTests.cs index 3f5a50f6..b35d0b26 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/AbLegacyArrayWriteSupportParityTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests/AbLegacyArrayWriteSupportParityTests.cs @@ -1,24 +1,48 @@ using Shouldly; using Xunit; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; +using ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.PlcFamilies; namespace ZB.MOM.WW.OtOpcUa.Driver.AbLegacy.Tests; /// -/// Pins 's AbLegacy row against the REAL array encoder -/// (), the same way the Modbus and S7 parity tests do: for every element -/// type the server must advertise an array node writable exactly when the driver encodes it. +/// Pins 's AbLegacy row against the REAL write path, the same way the Modbus, +/// S7 and Galaxy parity tests do: the server must advertise an array node writable exactly when +/// encodes a whole-array value for it, so a driver change that adds or drops +/// an array shape fails here until the table follows. /// /// -/// ⚠️ SKIPPED until follow-up F1 lands: the driver gained an array encoder in this lane (review 07 U10), but -/// (Core.Abstractions, not owned by the AbLegacy lane) still lists AbLegacy as -/// scalar-only, so every AbLegacy array node stays read-only and the encoder is unreachable from OPC UA. When F1 -/// adds the row, delete the Skip; this test then holds the table and the encoder together. +/// The driver is driven end to end — FromTagConfig, the transport's own WriteRefusal, the data-file +/// width check, the element conversion — against a fake runtime that records EncodeArray. The value written +/// is the CLR array the server hands the driver for the node's DataType (the node manager converts a numeric +/// client array to the node's element type first), sized to the node's declared length. /// [Trait("Category", "Unit")] public sealed class AbLegacyArrayWriteSupportParityTests { - public static TheoryData AllTypes() => new(Enum.GetValues()); + private const string Host = "ab://10.0.0.5/1,0"; + + // Tag.DataType name → the CLR array the server delivers for a 1-D write of that type. + private static Array ClrArray(string dataType, int n) => dataType switch + { + "Boolean" => Enumerable.Repeat(true, n).ToArray(), + "SByte" => Enumerable.Repeat((sbyte)1, n).ToArray(), + "Byte" => Enumerable.Repeat((byte)1, n).ToArray(), + "Int16" => Enumerable.Repeat((short)1, n).ToArray(), + "UInt16" => Enumerable.Repeat((ushort)1, n).ToArray(), + "Int32" => Enumerable.Repeat(1, n).ToArray(), + "UInt32" => Enumerable.Repeat(1u, n).ToArray(), + "Int64" => Enumerable.Repeat(1L, n).ToArray(), + "UInt64" => Enumerable.Repeat(1UL, n).ToArray(), + "Float" => Enumerable.Repeat(1f, n).ToArray(), + "Double" => Enumerable.Repeat(1d, n).ToArray(), + "String" => Enumerable.Repeat("a", n).ToArray(), + "DateTime" => Enumerable.Repeat(DateTime.UtcNow, n).ToArray(), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + private static readonly string[] NodeTypes = + ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64", "Float", "Double", "String", "DateTime"]; private static string AddressFor(AbLegacyDataType t) => t switch { @@ -32,24 +56,131 @@ public sealed class AbLegacyArrayWriteSupportParityTests _ => "N7:0", }; - [Theory(Skip = "F1: DriverArrayWriteSupport has no AbLegacy row yet (Core.Abstractions is outside this lane)")] - [MemberData(nameof(AllTypes))] - public void Table_agrees_with_the_array_encoder(AbLegacyDataType dataType) + private sealed class RecordingTag(AbLegacyTagCreateParams p) : FakeAbLegacyTag(p) + { + public Array? Encoded { get; private set; } + + public override void EncodeArray(AbLegacyDataType type, Array values) => Encoded = values; + } + + // The fake runtime, but the PRODUCTION transport's pre-wire write refusal (libplctag: translated names; native: + // PLC-5 and status-bit sub-elements). + private sealed class ParityFactory(IAbLegacyTagFactory transport) : IAbLegacyTagFactory + { + public List Created { get; } = []; + + public IAbLegacyTagRuntime Create(AbLegacyTagCreateParams p) + { + var tag = new RecordingTag(p); + lock (Created) Created.Add(tag); + return tag; + } + + public string? WriteRefusal(AbLegacyTagCreateParams p) => transport.WriteRefusal(p); + } + + // Whether the driver encodes a whole-array write of the node's declared length for this tag. + private static async Task DriverEncodesAsync( + string tagConfig, string nodeType, int nodeLength, AbLegacyPlcFamily family, bool native) + { + var factory = new ParityFactory(native ? new NativeAbLegacyTagFactory() : new LibplctagLegacyTagFactory()); + using var drv = new AbLegacyDriver(new AbLegacyDriverOptions + { + Devices = [new AbLegacyDeviceOptions(Host, family)], + RawTags = [new RawTagEntry("Arr", tagConfig, WriteIdempotent: false, DeviceName: Host)], + Probe = new AbLegacyProbeOptions { Enabled = false }, + }, "drv-parity", factory); + await drv.InitializeAsync("{}", CancellationToken.None); + + var r = await drv.WriteAsync([new WriteRequest("Arr", ClrArray(nodeType, nodeLength))], CancellationToken.None); + + bool encoded; + lock (factory.Created) encoded = factory.Created.Any(t => t.Encoded is { } a && a.Length == nodeLength && t.WriteCount == 1); + return r[0].StatusCode == AbLegacyStatusMapper.Good && encoded; + } + + public static TheoryData TypeMatrix() + { + var data = new TheoryData(); + foreach (var t in Enum.GetValues()) + foreach (var n in NodeTypes) + data.Add(t, n); + return data; + } + + /// Every element type × every node DataType, on the matching data file: the table answers exactly what + /// the write path does. + [Theory] + [MemberData(nameof(TypeMatrix))] + public async Task Table_agrees_with_the_write_path_for_every_element_and_node_type(AbLegacyDataType dataType, string nodeType) { var json = $$"""{"address":"{{AddressFor(dataType)}}","dataType":"{{dataType}}","isArray":true,"arrayLength":2}"""; - var encoderAccepts = AbLegacyTagDefinitionFactory.FromTagConfig(json, "p", out _) - && AbLegacyArrayWriteEncoder.TryConvert(dataType, new object[] { 1, 1 }, 2, out _) == AbLegacyStatusMapper.Good; - DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, "Int16", json).ShouldBe(encoderAccepts, dataType.ToString()); + var driver = await DriverEncodesAsync(json, nodeType, 2, AbLegacyPlcFamily.Slc500, native: false); + + DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, nodeType, json).ShouldBe(driver, $"{dataType} on a {nodeType} node"); } + /// + /// Shapes beyond the element type, both directions: the /bit, .sub, type-versus-file-width and + /// over-long cases the write path refuses, a native PLC-5 device (refused before the wire), and their positive + /// controls. Each row carries the address, element type, length, family and transport. + /// + [Theory] + [InlineData("N7:0", "Int", 4, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:0", "Int", 4, AbLegacyPlcFamily.Slc500, true)] + [InlineData("N7:0", "Int", 4, AbLegacyPlcFamily.MicroLogix, true)] + [InlineData("N7:0", "Int", 4, AbLegacyPlcFamily.Plc5, false)] // libplctag writes PLC-5 + [InlineData("N7:0", "Int", 4, AbLegacyPlcFamily.Plc5, true)] // native refuses every PLC-5 write + [InlineData("B3:0", "Int", 4, AbLegacyPlcFamily.Slc500, false)] + [InlineData("A9:0", "AnalogInt", 4, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:10", "AnalogInt", 4, AbLegacyPlcFamily.Slc500, false)] + [InlineData("L9:0", "Long", 4, AbLegacyPlcFamily.Slc500, false)] + [InlineData("F8:0", "Float", 4, AbLegacyPlcFamily.Slc500, true)] + [InlineData("N7:0", "Int", 256, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:0", "Int", 300, AbLegacyPlcFamily.Slc500, false)] // clamped to 256 by the factory: a 300-element write is refused + [InlineData("N7:0/3", "Int", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:0/3", "Bit", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("T4:0.ACC", "Int", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("F8:0", "Int", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:0", "Long", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("L9:0", "Int", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("N7:0", "Float", 2, AbLegacyPlcFamily.Slc500, false)] + [InlineData("S:0", "Int", 2, AbLegacyPlcFamily.Slc500, false)] // libplctag refuses a translated name + [InlineData("O:0", "Int", 2, AbLegacyPlcFamily.Slc500, false)] + public async Task Table_agrees_with_the_write_path_for_each_shape( + string address, string dataType, int length, AbLegacyPlcFamily family, bool native) + { + var json = $$"""{"address":"{{address}}","dataType":"{{dataType}}","isArray":true,"arrayLength":{{length}}}"""; + var driverConfig = native ? """{"transport":"Native"}""" : "{}"; + var deviceConfig = $$"""{"hostAddress":"{{Host}}","plcFamily":"{{family}}"}"""; + + var driver = await DriverEncodesAsync(json, "Int16", length, family, native); + + DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, "Int16", json, driverConfig, deviceConfig) + .ShouldBe(driver, $"{address} {dataType}[{length}] {family} native={native}"); + } + + /// + /// Shapes the table keeps read-only although the write path may encode them — the deliberately conservative + /// direction, pinned so it is a decision and not drift: an INDIRECT address (its element moves with the index + /// word, and the index read is not atomic with the write), and the file-number-less S: / I: / + /// O: files on the native transport (libplctag refuses them; the table does not read the transport + /// closely enough to widen output-image and status-file writes on native alone). + /// + [Theory] + [InlineData("""{"address":"N7:[N7:0]","dataType":"Int","isArray":true,"arrayLength":2}""", "{}")] + [InlineData("""{"address":"S:0","dataType":"Int","isArray":true,"arrayLength":2}""", """{"transport":"Native"}""")] + [InlineData("""{"address":"O:0","dataType":"Int","isArray":true,"arrayLength":2}""", """{"transport":"Native"}""")] + public void Table_keeps_the_conservative_shapes_read_only(string tagConfig, string driverConfig) => + DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, "Int16", tagConfig, driverConfig).ShouldBeFalse(); + /// /// The shapes the WRITE PATH refuses although their element type alone looks writable (review 07 r0 M6): a /// /bit or .sub address, an indirect address, a type that does not match its data file (I3), and - /// a native PLC-5 device. The F1 row must answer false for every one — keyed on dataType alone it would - /// advertise them writable. Skipped with the parity theory until F1 lands. + /// a native PLC-5 device. Keyed on dataType alone the row would advertise them writable. /// - [Theory(Skip = "F1: DriverArrayWriteSupport has no AbLegacy row yet (Core.Abstractions is outside this lane)")] + [Theory] [InlineData("""{"address":"N7:0/3","dataType":"Int","isArray":true,"arrayLength":2}""", "{}")] [InlineData("""{"address":"T4:0.ACC","dataType":"Int","isArray":true,"arrayLength":2}""", "{}")] [InlineData("""{"address":"N7:[N7:0]","dataType":"Int","isArray":true,"arrayLength":2}""", "{}")] @@ -60,7 +191,7 @@ public sealed class AbLegacyArrayWriteSupportParityTests public void Table_refuses_the_shapes_the_write_path_refuses(string tagConfig, string driverConfig) => DriverArrayWriteSupport.Supports(DriverTypeNames.AbLegacy, "Int16", tagConfig, driverConfig).ShouldBeFalse(); - /// What the encoder accepts today — the row F1 must add (Int, AnalogInt, Long, Float on a matching file). + /// What the encoder accepts: Int, AnalogInt, Long, Float (on a matching file). [Theory] [InlineData(AbLegacyDataType.Int, true)] [InlineData(AbLegacyDataType.AnalogInt, true)] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayWriteSupportParityTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayWriteSupportParityTests.cs new file mode 100644 index 00000000..fdf05c23 --- /dev/null +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests/TwinCATArrayWriteSupportParityTests.cs @@ -0,0 +1,135 @@ +using Shouldly; +using Xunit; +using ZB.MOM.WW.OtOpcUa.Core.Abstractions; + +namespace ZB.MOM.WW.OtOpcUa.Driver.TwinCAT.Tests; + +/// +/// Pins 's TwinCAT row against the REAL write path (#739, review 07 U9), the +/// same way the Modbus, S7 and Galaxy parity tests do: the server advertises an array node writable exactly when +/// converts a whole-array value (TwinCATArrayCodec.ToWriteElements) +/// and hands it to WriteArrayAsync, AND the production lays the +/// converted elements out. A driver change that adds or drops an array element type fails here until the table +/// follows. +/// +/// +/// The value written is the CLR array the server hands the driver for the node's DataType (the node manager +/// converts a numeric client array to the node's element type first). Every array node is materialised 1-D; a +/// multi-dimensional PLC array is written row-major from that flat value, so nothing here considers more than one +/// rank. The PLC-side shape check (CheckArray: declared length ≤ the PLC's element count, element width) +/// runs against the resolved symbol at write time and is outside what a compose-time table can know. +/// +[Trait("Category", "Unit")] +public sealed class TwinCATArrayWriteSupportParityTests +{ + private const string Host = "ads://5.23.91.23.1.1:851"; + + private static Array ClrArray(string dataType, int n) => dataType switch + { + "Boolean" => Enumerable.Repeat(true, n).ToArray(), + "SByte" => Enumerable.Repeat((sbyte)1, n).ToArray(), + "Byte" => Enumerable.Repeat((byte)1, n).ToArray(), + "Int16" => Enumerable.Repeat((short)1, n).ToArray(), + "UInt16" => Enumerable.Repeat((ushort)1, n).ToArray(), + "Int32" => Enumerable.Repeat(1, n).ToArray(), + "UInt32" => Enumerable.Repeat(1u, n).ToArray(), + "Int64" => Enumerable.Repeat(1L, n).ToArray(), + "UInt64" => Enumerable.Repeat(1UL, n).ToArray(), + "Float" => Enumerable.Repeat(1f, n).ToArray(), + "Double" => Enumerable.Repeat(1d, n).ToArray(), + // Non-numeric text: a String node over a non-string element converts only when the text happens to parse, + // which is not a write the server can promise — the table keeps it read-only. + "String" => Enumerable.Repeat("a", n).ToArray(), + "DateTime" => Enumerable.Repeat(new DateTime(2026, 10, 22, 12, 0, 0, DateTimeKind.Utc), n).ToArray(), + _ => throw new ArgumentOutOfRangeException(nameof(dataType)), + }; + + private static readonly string[] NodeTypes = + ["Boolean", "SByte", "Byte", "Int16", "UInt16", "Int32", "UInt32", "Int64", "UInt64", "Float", "Double", "String", "DateTime"]; + + // A PLC element width the codec can lay each type out at (STRING(80) / WSTRING(80) for the string types). + private static int StrideOf(TwinCATDataType t) => t switch + { + TwinCATDataType.String => 81, + TwinCATDataType.WString => 162, + _ => TwinCATTypeGuard.DeclaredByteSize(t), + }; + + private static async Task DriverEncodesAsync(string tagConfig, string nodeType, int count) + { + var fake = new FakeTwinCATClient(); + using var drv = new TwinCATDriver(new TwinCATDriverOptions + { + Devices = [new TwinCATDeviceOptions(Host)], + RawTags = [new RawTagEntry("Arr", tagConfig, WriteIdempotent: false, DeviceName: Host)], + Probe = new TwinCATProbeOptions { Enabled = false }, + }, "twincat-parity", new FakeTwinCATClientFactory { Customise = () => fake }); + await drv.InitializeAsync("{}", CancellationToken.None); + + var r = await drv.WriteAsync([new WriteRequest("Arr", ClrArray(nodeType, count))], CancellationToken.None); + if (r[0].StatusCode != TwinCATStatusMapper.Good || fake.ArrayWriteLog.Count != 1) return false; + + var (_, type, elements) = fake.ArrayWriteLog[0]; + if (elements.Count != count) return false; + try + { + TwinCATArrayCodec.Encode(type, elements, new TwinCATArrayLayout(count, StrideOf(type)), utf8: false); + return true; + } + catch (Exception ex) when (ex is NotSupportedException or InvalidCastException or FormatException or OverflowException) + { + return false; + } + } + + public static TheoryData TypeMatrix() + { + var data = new TheoryData(); + foreach (var t in Enum.GetValues()) + foreach (var n in NodeTypes) + data.Add(t, n); + return data; + } + + /// Every element type × every node DataType: the table answers exactly what the write path does. + [Theory] + [MemberData(nameof(TypeMatrix))] + public async Task Table_agrees_with_the_write_path_for_every_element_and_node_type(TwinCATDataType dataType, string nodeType) + { + var json = $$"""{"symbolPath":"MAIN.Arr","dataType":"{{dataType}}","isArray":true,"arrayLength":3}"""; + + if (dataType == TwinCATDataType.String && nodeType == "DateTime") + { + // The one CULTURE-dependent pair: a DateTime element reaches a STRING as Convert.ToString(DateTime), and + // whether its strict Windows-1252 encode accepts that text depends on the host culture (ICU's en-US puts a + // U+202F before "PM"; NLS a plain space). The table keeps it read-only — the conservative direction. + DriverArrayWriteSupport.Supports(DriverTypeNames.TwinCAT, nodeType, json).ShouldBeFalse(); + return; + } + + var driver = await DriverEncodesAsync(json, nodeType, 3); + + DriverArrayWriteSupport.Supports(DriverTypeNames.TwinCAT, nodeType, json).ShouldBe(driver, $"{dataType} on a {nodeType} node"); + } + + /// + /// Shapes beyond the element type, both directions: a BOOL bit-in-word path (a scalar bit write, which an array + /// value cannot feed), an undeclared length (a scalar tag), an element-offset path into a multi-dimensional + /// array, a TwinCAT 2 global, and their positive controls. + /// + [Theory] + [InlineData("""{"symbolPath":"MAIN.Flags","dataType":"Bool","isArray":true,"arrayLength":3}""", "Boolean")] + [InlineData("""{"symbolPath":"MAIN.Word.3","dataType":"Bool","isArray":true,"arrayLength":3}""", "Boolean")] + [InlineData("""{"symbolPath":"MAIN.Word.3","dataType":"Int","isArray":true,"arrayLength":3}""", "Int16")] + [InlineData("""{"symbolPath":"MAIN.Grid[1,0]","dataType":"Int","isArray":true,"arrayLength":3}""", "Int16")] + [InlineData("""{"symbolPath":".GlobalArr","dataType":"Real","isArray":true,"arrayLength":3}""", "Float")] + [InlineData("""{"symbolPath":"MAIN.Arr","dataType":"Int","isArray":true}""", "Int16")] + [InlineData("""{"symbolPath":"MAIN.Arr","dataType":"Int","isArray":true,"arrayLength":0}""", "Int16")] + [InlineData("""{"symbolPath":"MAIN.Arr","isArray":true,"arrayLength":3}""", "Int32")] // dataType ⇒ DInt + public async Task Table_agrees_with_the_write_path_for_each_shape(string tagConfig, string nodeType) + { + var driver = await DriverEncodesAsync(tagConfig, nodeType, 3); + + DriverArrayWriteSupport.Supports(DriverTypeNames.TwinCAT, nodeType, tagConfig).ShouldBe(driver, tagConfig); + } +} diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ArrayWriteWireTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ArrayWriteWireTests.cs index 3d8f0168..26934f4f 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ArrayWriteWireTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests/ArrayWriteWireTests.cs @@ -18,9 +18,10 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer.IntegrationTests; /// /// An S7 Int16 array (S7 encodes non-Bool arrays as one block) is writable on BOTH its raw node and /// its UNS mirror, and a whole-array write round-trips. -/// An AbCip DInt array (AbCip has a scalar-only write encoder) stays read-only even though it is -/// authored ReadWrite: the write is BadNotWritable at the node and never reaches the -/// gateway — the "never advertise a write the driver will refuse" rule. +/// An AbCip BOOL array (bit-packed DWORDs: AbCip's array encoder refuses it, though it encodes the atomic +/// numeric arrays since #737) stays read-only even though it is authored ReadWrite: the write is +/// BadNotWritable at the node and never reaches the gateway — the "never advertise a write the +/// driver will refuse" rule. /// A device-refused array write is reverted on whichever NodeId it was written through. /// A wrong-length array and an IndexRange (partial) write are refused inline, before dispatch, and /// leave the node holding its prior array. @@ -110,7 +111,8 @@ public sealed class ArrayWriteWireTests { TagId = "t-counts", DeviceId = "dev-2", Name = "Counts", DataType = "Int32", AccessLevel = TagAccessLevel.ReadWrite, - TagConfig = "{\"tagPath\":\"Counts\",\"dataType\":\"DInt\",\"isArray\":true,\"arrayLength\":4}", + // A BOOL array: the AbCip array encoder has no encoding for it (a DInt array is writable since #737). + TagConfig = "{\"tagPath\":\"Counts\",\"dataType\":\"Bool\",\"isArray\":true,\"arrayLength\":4}", }; var area = new UnsArea { UnsAreaId = "area-1", ClusterId = "c1", Name = "filling" }; var line = new UnsLine { UnsLineId = "line-1", UnsAreaId = "area-1", Name = "line-1" }; diff --git a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/AddressSpaceComposerDualNamespaceTests.cs b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/AddressSpaceComposerDualNamespaceTests.cs index 0698f2e8..fbbd41ac 100644 --- a/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/AddressSpaceComposerDualNamespaceTests.cs +++ b/tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/AddressSpaceComposerDualNamespaceTests.cs @@ -271,8 +271,8 @@ public sealed class AddressSpaceComposerDualNamespaceTests } /// Gitea #610: ArrayWritable is derived per OWNING DRIVER at compose time, and the UNS mirror - /// inherits it as BackingArrayWritable. An S7 Int16 array is writable-capable; the identical shape on - /// an AbCip driver (scalar-only write encoder) is not; a scalar is never flagged. + /// inherits it as BackingArrayWritable. An S7 Int16 array is writable-capable; an AbCip BOOL array (bit-packed + /// DWORDs, which AbCip's array encoder refuses) is not; a scalar is never flagged. [Fact] public void Array_write_capability_follows_the_owning_driver_and_reaches_the_uns_mirror() { @@ -286,7 +286,7 @@ public sealed class AddressSpaceComposerDualNamespaceTests { new Tag { TagId = "t1", DeviceId = "d1", Name = "Levels", DataType = "Int16", AccessLevel = TagAccessLevel.ReadWrite, TagConfig = s7Array }, new Tag { TagId = "t2", DeviceId = "d1", Name = "Level", DataType = "Int16", AccessLevel = TagAccessLevel.ReadWrite, TagConfig = "{\"address\":\"DB1.DBW8\",\"dataType\":\"Int16\"}" }, - new Tag { TagId = "t3", DeviceId = "d2", Name = "Counts", DataType = "Int32", AccessLevel = TagAccessLevel.ReadWrite, TagConfig = "{\"tagPath\":\"Counts\",\"dataType\":\"DInt\",\"isArray\":true,\"arrayLength\":4}" }, + new Tag { TagId = "t3", DeviceId = "d2", Name = "Counts", DataType = "Boolean", AccessLevel = TagAccessLevel.ReadWrite, TagConfig = "{\"tagPath\":\"Counts\",\"dataType\":\"Bool\",\"isArray\":true,\"arrayLength\":4}" }, }; var area = new UnsArea { UnsAreaId = "a", ClusterId = "c1", Name = "filling" }; var line = new UnsLine { UnsLineId = "l", UnsAreaId = "a", Name = "line-1" }; @@ -303,7 +303,7 @@ public sealed class AddressSpaceComposerDualNamespaceTests c.RawTags.Single(t => t.Name == "Levels").ArrayWritable.ShouldBeTrue(); c.RawTags.Single(t => t.Name == "Level").ArrayWritable.ShouldBeFalse(); // scalar: never flagged - c.RawTags.Single(t => t.Name == "Counts").ArrayWritable.ShouldBeFalse(); // AbCip: no array encoder + c.RawTags.Single(t => t.Name == "Counts").ArrayWritable.ShouldBeFalse(); // AbCip: no BOOL-array encoder c.UnsReferenceVariables.Single(v => v.UnsTagReferenceId == "r1").BackingArrayWritable.ShouldBeTrue(); c.UnsReferenceVariables.Single(v => v.UnsTagReferenceId == "r3").BackingArrayWritable.ShouldBeFalse(); @@ -347,4 +347,71 @@ public sealed class AddressSpaceComposerDualNamespaceTests c.RawTags.Single(t => t.Name == "overDeviceCap").ArrayWritable.ShouldBeFalse(); c.UnsReferenceVariables.Single().BackingArrayWritable.ShouldBeFalse(); } + + /// + /// #737 / #738 / #739 (review 07 U10 / U9): AbCip, AbLegacy and TwinCAT encode whole-array writes, so an + /// authored-ReadWrite array tag on each now materialises WRITABLE — raw node and UNS mirror alike — with + /// the ScadaBridge-shaped Int16 node type. An array already deployed read-only turns writable on the next deploy + /// through ArrayWritable, which rides the surgical whitelist (pinned in the classifier tests). A shape + /// each encoder refuses (an AbCip BOOL array, an AbLegacy Bit array, an AbLegacy Int array on an F file, a + /// TwinCAT BOOL bit-in-word path) stays read-only on the SAME drivers — the negative control. + /// + [Fact] + public void AbCip_AbLegacy_and_TwinCAT_arrays_materialise_writable_and_refused_shapes_stay_read_only() + { + var folder = new RawFolder { RawFolderId = "f", ClusterId = "c1", Name = "Plant" }; + var ab = new DriverInstance { DriverInstanceId = "drv-ab", ClusterId = "c1", RawFolderId = "f", Name = "Ab", DriverType = "AbCip", DriverConfig = "{}" }; + var slc = new DriverInstance { DriverInstanceId = "drv-slc", ClusterId = "c1", RawFolderId = "f", Name = "Slc", DriverType = "AbLegacy", DriverConfig = "{}" }; + var tc = new DriverInstance { DriverInstanceId = "drv-tc", ClusterId = "c1", RawFolderId = "f", Name = "Tc", DriverType = "TwinCAT", DriverConfig = "{}" }; + var d1 = new Device { DeviceId = "d1", DriverInstanceId = "drv-ab", Name = "plc", DeviceConfig = "{}" }; + var d2 = new Device { DeviceId = "d2", DriverInstanceId = "drv-slc", Name = "slc", DeviceConfig = "{\"plcFamily\":\"Slc500\"}" }; + var d3 = new Device { DeviceId = "d3", DriverInstanceId = "drv-tc", Name = "ads", DeviceConfig = "{}" }; + static Tag Arr(string id, string device, string dataType, string tagConfig) => new() + { + TagId = id, DeviceId = device, Name = id, DataType = dataType, AccessLevel = TagAccessLevel.ReadWrite, + TagConfig = tagConfig, + }; + var tags = new[] + { + Arr("abInts", "d1", "Int16", "{\"tagPath\":\"Ints\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":4}"), + Arr("abBools", "d1", "Boolean", "{\"tagPath\":\"Bools\",\"dataType\":\"Bool\",\"isArray\":true,\"arrayLength\":4}"), + Arr("slcInts", "d2", "Int16", "{\"address\":\"N7:0\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":4}"), + Arr("slcBits", "d2", "Boolean", "{\"address\":\"B3:0\",\"dataType\":\"Bit\",\"isArray\":true,\"arrayLength\":4}"), + Arr("slcWrongFile", "d2", "Int16", "{\"address\":\"F8:0\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":4}"), + Arr("tcInts", "d3", "Int16", "{\"symbolPath\":\"MAIN.Ints\",\"dataType\":\"Int\",\"isArray\":true,\"arrayLength\":4}"), + Arr("tcBit", "d3", "Boolean", "{\"symbolPath\":\"MAIN.Word.3\",\"dataType\":\"Bool\",\"isArray\":true,\"arrayLength\":4}"), + }; + var area = new UnsArea { UnsAreaId = "a", ClusterId = "c1", Name = "filling" }; + var line = new UnsLine { UnsLineId = "l", UnsAreaId = "a", Name = "line-1" }; + var equip = new Equipment { EquipmentId = "e", UnsLineId = "l", Name = "station-1", MachineCode = "S1" }; + var refs = new[] + { + new UnsTagReference { UnsTagReferenceId = "r-ab", EquipmentId = "e", TagId = "abInts" }, + new UnsTagReference { UnsTagReferenceId = "r-slc", EquipmentId = "e", TagId = "slcInts" }, + new UnsTagReference { UnsTagReferenceId = "r-tc", EquipmentId = "e", TagId = "tcInts" }, + new UnsTagReference { UnsTagReferenceId = "r-bits", EquipmentId = "e", TagId = "slcBits" }, + }; + + var c = AddressSpaceComposer.Compose( + new[] { area }, new[] { line }, new[] { equip }, new[] { ab, slc, tc }, Array.Empty(), + unsTagReferences: refs, tags: tags, rawFolders: new[] { folder }, devices: new[] { d1, d2, d3 }); + + foreach (var writable in new[] { "abInts", "slcInts", "tcInts" }) + { + var plan = c.RawTags.Single(t => t.Name == writable); + plan.ArrayWritable.ShouldBeTrue(writable); + AddressSpaceApplier.RawNodeWritable(plan).ShouldBeTrue(writable); + } + foreach (var refused in new[] { "abBools", "slcBits", "slcWrongFile", "tcBit" }) + { + var plan = c.RawTags.Single(t => t.Name == refused); + plan.IsArray.ShouldBeTrue(refused); // still an array node — just not a writable one + plan.ArrayWritable.ShouldBeFalse(refused); + AddressSpaceApplier.RawNodeWritable(plan).ShouldBeFalse(refused); + } + + foreach (var id in new[] { "r-ab", "r-slc", "r-tc" }) + AddressSpaceApplier.UnsReferenceNodeWritable(c.UnsReferenceVariables.Single(v => v.UnsTagReferenceId == id)).ShouldBeTrue(id); + AddressSpaceApplier.UnsReferenceNodeWritable(c.UnsReferenceVariables.Single(v => v.UnsTagReferenceId == "r-bits")).ShouldBeFalse(); + } } From c1b9c7ec5cb97598724eaac8a9fc5f84748c8781 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:32:23 -0400 Subject: [PATCH 18/20] fix(sql): attribute a duplicated key to its own tags; key-value cap gets 2x headroom (R06:S-1 review I-1, m-3) One stray duplicate row used to fail every tag in a key-value group or none of them, depending on whether an unrelated key was absent. Duplicated keys are now refused per key: only that key's tags publish BadConfigurationError and the rest of the group reads normally. The key-value cap reads up to 2x the distinct keys (+1 to detect excess), so a few stray duplicates are attributed per key and only a gross excess (history/events table) fails the whole group. Duplicated-key refusals count as answered refusals for health. The row-cap, duplicate-key and multi-row warnings now use the per-plan limiter (m-3); the reader-wide limiter is removed. Unit tests pin both the all-keys-present and absent-key cases with Good controls; live SQL Server gate added (23/23). Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Sql.md | 27 +++-- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 2 +- .../SqlGroupPlanner.cs | 10 +- .../SqlPollReader.cs | 104 ++++++++++-------- .../SqlQueryPlan.cs | 12 +- .../SqlServerReadTests.cs | 63 +++++++++++ .../SqlGroupPlannerTests.cs | 17 +-- .../SqlPollReaderTests.cs | 56 ++++++++-- 8 files changed, 206 insertions(+), 85 deletions(-) diff --git a/docs/drivers/Sql.md b/docs/drivers/Sql.md index ffa68922..120d41c1 100644 --- a/docs/drivers/Sql.md +++ b/docs/drivers/Sql.md @@ -166,16 +166,23 @@ emits a `SqlQueryPlan` per distinct group key. Grouping is deterministic (groups order of their first member, members keep their input order, parameters keep first-appearance order), so an unchanged configuration produces byte-identical SQL. -**Every key-value and where-pair query is row-capped (R06:S-1).** A key-value plan asks for one row more -than it has distinct keys (`TOP (n+1)` / `LIMIT n+1` / `FETCH FIRST n+1 ROWS ONLY`), a where-pair wide-row plan -for two, and the reader stops reading there even if a provider ignores the limit. Receiving the extra row -proves the selector is ambiguous — a history/events table, or a where column that is not unique — so every -member of that group publishes `BadConfigurationError` with a rate-limited warning naming the table and -selector, instead of the old behaviour of materialising every matching row on every poll and publishing an -arbitrary "last row" at Good. A duplicate that fits inside the cap (a duplicated key alongside an absent one) -still resolves last-row-wins. `topByTimestamp` keeps its single-row limit. An over-cap answer, like every -other refusal (absent row, NULL cell, uncoercible cell), is the database answering: it advances -`LastSuccessfulRead` and restores `Healthy` after a comms fault. +**Every key-value and where-pair query is row-capped (R06:S-1).** A key-value plan reads at most twice as +many rows as it has distinct keys and asks for one more (`TOP (2n+1)` / `LIMIT 2n+1` / `FETCH FIRST 2n+1 ROWS +ONLY`); a where-pair wide-row plan asks for two. The reader stops there even if a provider ignores the limit. + +- **A duplicated key is refused per key.** When a key comes back more than once, only the tags on THAT key + publish `BadConfigurationError`; every other key in the group reads normally, whatever else is present or + absent. (Before this, a duplicate was last-row-wins Good.) +- **A gross excess fails the group.** Receiving the extra row past the cap means a history/events table or a + non-unique where column, which cannot be attributed because the rest is unread, so every member of the group + publishes `BadConfigurationError`. For a where-pair any second row is this case. +- Each case logs a per-plan rate-limited warning naming the table and the selector (or an example key). + `topByTimestamp` keeps its single-row limit. +- **Upgrade note:** a deployment whose key-value source carries stray duplicate rows, or whose where column is + not unique, will see those tags go Bad where they used to read an arbitrary row at Good. + +Every such refusal, like the other refusals (absent row, NULL cell, uncoercible cell), is the database +answering: it advances `LastSuccessfulRead` and restores `Healthy` after a comms fault. ⚠️ **Group keys are case-sensitive on identifiers.** Two tags authored `dbo.T` and `DBO.T` plan as **two** groups — one extra round-trip, not a correctness problem, but worth knowing if a group count diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index 7fbd99d7..d7212245 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -542,7 +542,7 @@ public sealed class SqlDriver try { var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); - ObservePollOutcome(result.Snapshots, result.RowCapExceededGroups, generation); + ObservePollOutcome(result.Snapshots, result.AnsweredRefusalGroups, generation); return result.Snapshots; } catch (OperationCanceledException) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs index 31c77351..915eb89c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlGroupPlanner.cs @@ -230,10 +230,12 @@ public static class SqlGroupPlanner values.Add(keyValue); } - // Row cap (R06:S-1): one row per distinct key is all this plan can use, so it asks for ONE more — enough - // to see a source breaking the one-row-per-key contract, never enough to pull a history table's every - // matching row into memory on every poll. - var rowCap = values.Count; + // Row cap (R06:S-1): one row per distinct key is all this plan can use. It reads up to TWICE that, so a + // handful of stray duplicates stays inside the cap and is attributed PER KEY by the reader (only the + // duplicated key's tags go BadConfigurationError) — and asks for one row more than that, so only a + // gross excess (a history/events table, unattributable by construction because the rest is unread) + // fails the whole group. Memory stays O(keys) either way. + var rowCap = values.Count * 2; var sql = new StringBuilder("SELECT ") .Append(dialect.RowLimitPrefix(rowCap + 1)) .Append(QuoteList(selected, dialect)) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index fdb22ff0..ffeb3804 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -68,7 +68,6 @@ public sealed class SqlPollReader : IDisposable private int _disposed; - private long _lastContractWarningTicks; /// /// Per-key last-warned instant for the per-poll warnings (R06:S-6): a frozen group's BadTimeout and a @@ -215,7 +214,7 @@ public sealed class SqlPollReader : IDisposable results[i] = new DataValueSnapshot(null, SqlStatusCodes.BadNodeIdUnknown, null, readAt); } - var rowCapExceeded = shape.Definitions.Count > 0 + var answeredRefusals = shape.Definitions.Count > 0 ? await ReadGroupsAsync(shape, results, readAt, cancellationToken).ConfigureAwait(false) : 0; @@ -224,7 +223,7 @@ public sealed class SqlPollReader : IDisposable for (var i = 0; i < results.Length; i++) results[i] ??= new DataValueSnapshot(null, SqlStatusCodes.BadInternalError, null, readAt); - return new SqlPollResult(results, rowCapExceeded); + return new SqlPollResult(results, answeredRefusals); } /// How many times a reference list has been (re)planned. Test observation of the S-7 cache. @@ -301,7 +300,7 @@ public sealed class SqlPollReader : IDisposable /// /// Runs every planned group concurrently and slices the results back. Returns how many groups came - /// back over their row cap. + /// back with an answered refusal (over the row cap, or a duplicated key). /// private async Task ReadGroupsAsync( PollShape shape, @@ -334,14 +333,14 @@ public sealed class SqlPollReader : IDisposable // leaves a sibling group running against a connection nobody is watching. var outcomes = await Task.WhenAll(groups).ConfigureAwait(false); - var rowCapExceeded = 0; + var answeredRefusals = 0; for (var g = 0; g < plans.Count; g++) { - Slice(plans[g], outcomes[g], slots, results, readAt); - if (ReferenceEquals(outcomes[g], GroupResult.RowCapExceeded)) rowCapExceeded++; + var refusedSome = Slice(plans[g], outcomes[g], slots, results, readAt); + if (refusedSome || ReferenceEquals(outcomes[g], GroupResult.RowCapExceeded)) answeredRefusals++; } - return rowCapExceeded; + return answeredRefusals; } /// @@ -622,14 +621,16 @@ public sealed class SqlPollReader : IDisposable if (plan.RowCap is { } rowCap && rows.Count > rowCap) { - if (ShouldWarnAboutContract()) + // Per plan, not reader-wide: with two ambiguous groups the reader-wide limiter could name only the + // first table forever, and with health still Healthy this line is the operator's only pointer. + if (ShouldWarn("rowcap:" + plan.SqlText)) { var first = plan.Members[0]; var selector = plan.Model == SqlTagModel.KeyValue - ? string.Concat(first.KeyColumn, " IN (", rowCap.ToString(CultureInfo.InvariantCulture), " key(s))") + ? string.Concat(first.KeyColumn, " IN (", (rowCap / 2).ToString(CultureInfo.InvariantCulture), " key(s))") : string.Concat(first.RowSelectorColumn, " = ", first.RowSelectorValue); _logger.LogWarning( - "Sql poll source '{Table}' returned more rows than the {Model} selector '{Selector}' can use " + + "Sql poll source '{Table}' returned more rows than the {Model} selector '{Selector}' can attribute " + "(at most {Cap}) in one poll, so the selector is ambiguous — a history/events table, or a " + "where column that is not unique. Its {Members} tag(s) publish BadConfigurationError and " + "nothing past row {Limit} is read. Point the tag at a current-values table or a latest-row view.", @@ -658,8 +659,11 @@ public sealed class SqlPollReader : IDisposable } } - /// Maps one group's outcome onto every slot its members occupy in the caller's list. - private void Slice( + /// + /// Maps one group's outcome onto every slot its members occupy in the caller's list. Returns whether any + /// member was refused on an ANSWERED result (a duplicated key) — answer evidence the driver must count. + /// + private bool Slice( SqlQueryPlan plan, GroupResult outcome, Dictionary> slots, @@ -669,9 +673,11 @@ public sealed class SqlPollReader : IDisposable // KeyValue indexes rows by the key column; WideRow's members all read the one selected row, so the // where-column is not even in the result set (design §5.3). Both selections happen ONCE per group — // they are group-wide facts, and doing them per member would report a contract violation N times. + HashSet? duplicatedKeys = null; var byKey = outcome.Succeeded && plan.Model == SqlTagModel.KeyValue - ? IndexByKey(plan, outcome) + ? IndexByKey(plan, outcome, out duplicatedKeys) : null; + var refusedSome = false; var wideRow = outcome.Succeeded && plan.Model == SqlTagModel.WideRow ? SelectWideRow(plan, outcome) : null; @@ -681,26 +687,45 @@ public sealed class SqlPollReader : IDisposable // Members may repeat (two tags on one keyValue, or the same ref twice) — every occurrence is fed. if (!slots.TryGetValue(member, out var positions)) continue; - var snapshot = outcome.Succeeded - ? MapMember(plan, member, outcome, byKey, wideRow, readAt) - : new DataValueSnapshot(null, outcome.FailureStatus, null, readAt); + DataValueSnapshot snapshot; + if (!outcome.Succeeded) + { + snapshot = new DataValueSnapshot(null, outcome.FailureStatus, null, readAt); + } + else if (duplicatedKeys is not null && member.KeyValue is { } key && duplicatedKeys.Contains(key)) + { + // More than one row for THIS tag's key: its selector is ambiguous — the same rule a where-pair + // follows — so it is a configuration error for this key's tags only. Every other key in the + // group is unaffected, whatever else is present or absent (R06:S-1 review I-1: it used to be + // last-row-wins Good, or the whole group Bad, depending on an unrelated key's absence). + snapshot = new DataValueSnapshot(null, SqlStatusCodes.BadConfigurationError, null, readAt); + refusedSome = true; + } + else + { + snapshot = MapMember(plan, member, outcome, byKey, wideRow, readAt); + } foreach (var position in positions) results[position] = snapshot; } + + return refusedSome; } /// - /// Builds the key → row index a key-value plan slices through. Later rows win, so a source that - /// breaks the one-row-per-key contract (design §3.6) still yields a deterministic value; the - /// violation is reported through a rate-limited warning rather than silently. + /// Builds the key → row index a key-value plan slices through, and the set of keys that came back more + /// than once — a source breaking the one-row-per-key contract (design §3.6). Those keys' tags publish + /// ; the violation is also reported through a + /// per-plan rate-limited warning. /// - private Dictionary IndexByKey(SqlQueryPlan plan, GroupResult outcome) + private Dictionary IndexByKey( + SqlQueryPlan plan, GroupResult outcome, out HashSet? duplicatedKeys) { + duplicatedKeys = null; var index = new Dictionary(StringComparer.Ordinal); if (plan.KeyColumn is null || !outcome.ColumnIndex.TryGetValue(plan.KeyColumn, out var keyAt)) return index; - var duplicated = 0; foreach (var row in outcome.Rows) { // A NULL key cell indexes nothing: Convert.ToString would fold it to "", which would then be @@ -711,20 +736,16 @@ public sealed class SqlPollReader : IDisposable // (an INTEGER station id, say) — the provider coerces on the way in, so match on the way out. var key = Convert.ToString(keyCell, CultureInfo.InvariantCulture); if (key is null) continue; - if (!index.TryAdd(key, row)) - { - index[key] = row; - duplicated++; - } + if (!index.TryAdd(key, row)) (duplicatedKeys ??= new HashSet(StringComparer.Ordinal)).Add(key); } - if (duplicated > 0 && ShouldWarnAboutContract()) + if (duplicatedKeys is not null && ShouldWarn("dupkey:" + plan.SqlText)) { _logger.LogWarning( - "Sql poll source '{Table}' returned more than one row for {Count} key(s) in one poll; the " + - "last row wins. The key-value model requires one row per key — point the tag at a " + - "current-values table or an operator-authored latest-per-key view.", - plan.Members[0].Table, duplicated); + "Sql poll source '{Table}' returned more than one row for {Count} key(s) in one poll (e.g. " + + "'{Example}'); those keys' tags publish BadConfigurationError. The key-value model requires one " + + "row per key — point the tag at a current-values table or an operator-authored latest-per-key view.", + plan.Members[0].Table, duplicatedKeys.Count, duplicatedKeys.First()); } return index; @@ -759,7 +780,7 @@ public sealed class SqlPollReader : IDisposable private object?[]? SelectWideRow(SqlQueryPlan plan, GroupResult outcome) { if (outcome.Rows.Count == 0) return null; - if (outcome.Rows.Count > 1 && ShouldWarnAboutContract()) + if (outcome.Rows.Count > 1 && ShouldWarn("multirow:" + plan.SqlText)) { var first = plan.Members[0]; var selector = string.IsNullOrWhiteSpace(first.RowSelectorColumn) @@ -920,15 +941,6 @@ public sealed class SqlPollReader : IDisposable } } - /// Rate-limits the source-contract warning so a misconfigured table cannot flood the log. - private bool ShouldWarnAboutContract() - { - var now = Stopwatch.GetTimestamp(); - var last = Interlocked.Read(ref _lastContractWarningTicks); - if (last != 0 && Stopwatch.GetElapsedTime(last, now) < ContractWarningInterval) return false; - return Interlocked.CompareExchange(ref _lastContractWarningTicks, now, last) == last; - } - /// One group's outcome: either a materialised result set, or the status its members inherit. private sealed class GroupResult { @@ -993,11 +1005,11 @@ public sealed class SqlPollReader : IDisposable /// One poll's snapshots plus the answer evidence they cannot carry themselves. /// One snapshot per requested reference, at the same index. -/// -/// Groups the database answered with more rows than their plan can use; their members carry -/// . +/// +/// Groups the database ANSWERED but whose result the reader refused in whole or part — an over-cap result +/// set, or a duplicated key. Those members carry . /// -public sealed record SqlPollResult(IReadOnlyList Snapshots, int RowCapExceededGroups); +public sealed record SqlPollResult(IReadOnlyList Snapshots, int AnsweredRefusalGroups); /// /// The OPC UA status codes the Sql driver publishes, and the quality-class predicates its tests read diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs index 333a762c..ec32c5d1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlQueryPlan.cs @@ -56,11 +56,13 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Sql; /// reader's map from result-set ordinal to source column. /// /// -/// The most rows this plan can USE — its distinct key count (key-value) or 1 (a where-pair wide row); -/// when the statement already limits itself (topByTimestamp). -/// asks for RowCap + 1, and the reader stops there: receiving the extra row proves the selector is -/// ambiguous (a history table, a non-unique where column) and the group publishes -/// BadConfigurationError instead of materialising every matching row on every poll (R06:S-1). +/// The most rows the reader will attribute before refusing the whole group — twice the distinct key count +/// (key-value: headroom so stray duplicates are attributed per key) or 1 (a where-pair wide row: one +/// selector, so any second row is the group's ambiguity); when the statement +/// already limits itself (topByTimestamp). asks for RowCap + 1, and the +/// reader stops there: receiving the extra row means a history/events table or a non-unique where column, +/// and the group publishes BadConfigurationError instead of materialising every matching row on +/// every poll (R06:S-1). /// public sealed record SqlQueryPlan( SqlTagModel Model, diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs index ff6271bc..73339481 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.IntegrationTests/SqlServerReadTests.cs @@ -116,6 +116,69 @@ public sealed class SqlServerReadTests } } + /// + /// Review I-1, live: one stray duplicate row in a key-value table fails ONLY that key's tag; the other + /// key in the same group (one capped SELECT TOP (5) … IN (@k0, @k1)) reads Good — the positive + /// control — and the driver stays Healthy. + /// + [Fact] + public async Task ReadAsync_realSqlServer_aDuplicatedKeyFailsOnlyItsOwnTag() + { + SkipUnlessLive(); + const string table = "RowCapCurrent"; + var qualified = $"{SqlPollServerFixture.Schema}.{table}"; + + await using (var arrange = await _sql.OpenInspectionConnectionAsync()) + { + await using var command = arrange.CreateCommand(); + command.CommandText = $""" + IF OBJECT_ID(N'{qualified}') IS NOT NULL DROP TABLE {qualified}; + CREATE TABLE {qualified} ( + {SqlPollServerFixture.KeyColumn} nvarchar(64) NOT NULL, + {SqlPollServerFixture.ValueColumn} float NULL, + {SqlPollServerFixture.TimestampColumn} datetime2 NOT NULL); + INSERT INTO {qualified} VALUES + (N'Pump7', 1.0, '2026-07-24T10:00:00'), + (N'Pump7', 2.0, '2026-07-24T10:00:01'), + (N'Pump8', 8.5, '2026-07-24T10:00:00'); + """; + await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } + + try + { + await using var driver = await StartAsync( + KeyValueOn("Sql/Current/Pump7", "Pump7", table), + KeyValueOn("Sql/Current/Pump8", "Pump8", table)); + + var snapshots = await driver.ReadAsync( + ["Sql/Current/Pump7", "Sql/Current/Pump8"], TestContext.Current.CancellationToken); + + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.Good); + Convert.ToDouble(snapshots[1].Value).ShouldBe(8.5); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + finally + { + await using var cleanup = await _sql.OpenInspectionConnectionAsync(); + await using var drop = cleanup.CreateCommand(); + drop.CommandText = $"DROP TABLE IF EXISTS {qualified}"; + await drop.ExecuteNonQueryAsync(CancellationToken.None); + } + } + + private static RawTagEntry KeyValueOn(string rawPath, string key, string table) => Tag(rawPath, new JsonObject + { + ["driver"] = "Sql", + ["model"] = "KeyValue", + ["table"] = SqlPollServerFixture.Qualified(table), + ["keyColumn"] = SqlPollServerFixture.KeyColumn, + ["keyValue"] = key, + ["valueColumn"] = SqlPollServerFixture.ValueColumn, + ["timestampColumn"] = SqlPollServerFixture.TimestampColumn, + }); + // ---- key-value model ---- /// The plan's headline case: a seeded key-value row round-trips as a Good snapshot. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs index 058ea516..ef7661b9 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlGroupPlannerTests.cs @@ -32,8 +32,9 @@ public class SqlGroupPlannerTests // ---- R06:S-1: the row cap ---- /// - /// A key-value plan asks for one row more than it has distinct keys, and a where-pair wide-row plan for - /// two: enough to SEE an ambiguous selector, never enough to materialise a history table every poll. + /// A key-value plan reads up to twice its distinct keys (so stray duplicates are attributed per key) and + /// asks for one row more; a where-pair wide-row plan asks for two. Enough to SEE an ambiguous selector, + /// never enough to materialise a history table every poll. /// The topByTimestamp form keeps its single-row limit and needs no cap. /// [Fact] @@ -49,8 +50,8 @@ public class SqlGroupPlannerTests ], new SqlServerDialect()).ToList(); var kv = plans.Single(p => p.Model == SqlTagModel.KeyValue); - kv.RowCap.ShouldBe(2); - kv.SqlText.ShouldStartWith("SELECT TOP (3) "); + kv.RowCap.ShouldBe(4); // 2 distinct keys, with 2x headroom for per-key attribution + kv.SqlText.ShouldStartWith("SELECT TOP (5) "); var wherePair = plans.Single(p => p.Parameters.Count == 1 && p.Model == SqlTagModel.WideRow); wherePair.RowCap.ShouldBe(1); @@ -90,7 +91,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT TOP (2) [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (3) [tag_name], [num_value], [sample_ts] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].ParameterNames.ShouldBe(new[] { "@k0" }); plans[0].SelectedColumns.ShouldBe(new[] { "tag_name", "num_value", "sample_ts" }); plans[0].Model.ShouldBe(SqlTagModel.KeyValue); @@ -107,7 +108,7 @@ public class SqlGroupPlannerTests new SqlServerDialect()); plans[0].SqlText.ShouldBe( - "SELECT TOP (2) [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (3) [tag_name], [num_value] FROM [dbo].[TagValues] WHERE [tag_name] IN (@k0)"); plans[0].TimestampColumn.ShouldBeNull(); } @@ -233,7 +234,7 @@ public class SqlGroupPlannerTests KvTag("B", "TagValues", "tag_name", "Line1.Temp", "num_value", null), ], new ColonMarkerDialect()).ToList(); - plans[0].SqlText.ShouldEndWith("IN (:k0, :k1) LIMIT 3"); + plans[0].SqlText.ShouldEndWith("IN (:k0, :k1) LIMIT 5"); plans[0].SqlText.ShouldNotContain("@"); plans[0].ParameterNames.ShouldBe(new[] { ":k0", ":k1" }); } @@ -456,7 +457,7 @@ public class SqlGroupPlannerTests // A column name is an identifier — it cannot be parameterized, so the guarantee is that it is // bracket-quoted with ] doubled, leaving one (nonexistent) identifier rather than executable SQL. plans[0].SqlText.ShouldBe( - "SELECT TOP (2) [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); + "SELECT TOP (3) [tag_name], [v]] ; DROP TABLE Users --] FROM [dbo].[T] WHERE [tag_name] IN (@k0)"); } // ---- contract edges ---- diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs index d26bd0b6..19d28944 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlPollReaderTests.cs @@ -272,25 +272,59 @@ public sealed class SqlPollReaderTests snapshots[0].SourceTimestampUtc!.Value.ShouldBe(snapshots[0].ServerTimestampUtc); } + /// + /// R06:S-1 review I-1. One stray duplicate row is attributed to ITS key: that key's tags publish + /// and every other key in the group reads normally. + /// It used to be last-row-wins Good, or the whole group Bad, depending on whether an UNRELATED key was + /// absent. Both variants are pinned: every other key present, and another key absent. The other keys' + /// Good / BadNoData are the positive controls. + /// + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ReadAsync_aDuplicatedKey_failsOnlyThatKeysTags_whateverElseIsAbsent(bool withAnAbsentKey) + { + using var fixture = new SqlitePollFixture(); + var logger = new RecordingLogger(); + fixture.Execute( + $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + + $"VALUES ('{SqlitePollFixture.PresentKey}', 99.0, '2026-07-24T10:00:09Z')"); + var tags = new List + { + KvTag("Speed", SqlitePollFixture.PresentKey), + KvTag("Speed2", SqlitePollFixture.PresentKey), // a second tag on the duplicated key + KvTag("Pressure", SqlitePollFixture.SecondPresentKey), + }; + if (withAnAbsentKey) tags.Add(KvTag("Gone", SqlitePollFixture.AbsentKey)); + var reader = Reader(fixture, logger, [.. tags]); + + var snapshots = await reader.ReadAsync([.. tags.Select(t => t.Name)], CancellationToken.None); + + snapshots[0].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.BadConfigurationError); + snapshots[2].StatusCode.ShouldBe(SqlStatusCodes.Good); + snapshots[2].Value.ShouldBe(SqlitePollFixture.SecondPresentValue); + if (withAnAbsentKey) snapshots[3].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); + + var warning = logger.Entries.ShouldHaveSingleItem(); + warning.Message.ShouldContain(SqlitePollFixture.PresentKey); + } + + /// The driver counts a duplicated-key refusal as the database answering, like an over-cap one. [Fact] - public async Task ReadAsync_whenASourceViolatesOneRowPerKey_takesTheLastRowDeterministically() + public async Task ReadDetailedAsync_aDuplicatedKey_isReportedAsAnAnsweredRefusal() { using var fixture = new SqlitePollFixture(); fixture.Execute( $"INSERT INTO {SqlitePollFixture.KeyValueTable} " + $"({SqlitePollFixture.KeyColumn}, {SqlitePollFixture.ValueColumn}, {SqlitePollFixture.TimestampColumn}) " + $"VALUES ('{SqlitePollFixture.PresentKey}', 99.0, '2026-07-24T10:00:09Z')"); - // Two keys, one absent: the duplicate fits inside the row cap (2), so it is not provably a history - // table — the last-row-wins rule still applies there (the cap test below covers the other side). - var reader = Reader(fixture, - KvTag("Speed", SqlitePollFixture.PresentKey), KvTag("Gone", SqlitePollFixture.AbsentKey)); + var reader = Reader(fixture, KvTag("Speed", SqlitePollFixture.PresentKey)); + var clean = Reader(fixture, KvTag("Pressure", SqlitePollFixture.SecondPresentKey)); - var snapshots = await reader.ReadAsync(["Speed", "Gone"], CancellationToken.None); - - // Design §3.6: the contract is one row per key; when a source breaks it the reader is at least - // deterministic — last occurrence in reader order — rather than silently arbitrary. - snapshots[0].Value.ShouldBe(99.0); - snapshots[1].StatusCode.ShouldBe(SqlStatusCodes.BadNoData); + (await reader.ReadDetailedAsync(["Speed"], CancellationToken.None)).AnsweredRefusalGroups.ShouldBe(1); + (await clean.ReadDetailedAsync(["Pressure"], CancellationToken.None)).AnsweredRefusalGroups.ShouldBe(0); } /// From 099f253321d7c8d308d7105927798954e65e65ef Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:35:52 -0400 Subject: [PATCH 19/20] fix(sql): review minors - superseded-reader polls, rebind faults, disposed-reader guard, JSON position (R06 G2 review m-1 m-2 m-4 m-6 m-7) m-1: a poll that captured a reader a rebind then disposed retries once on the current reader; with no newer reader the typed SqlPollReaderDisposedException is marked classified, so HandlePollError no longer degrades for it. Groups report a disposed reader as a sentinel so WhenAll surfaces the typed exception, not a bare cancel. m-2: TryAddRef refuses new groups once Dispose ran, matching the doc comment. m-4: a rebind / tag-table failure in InitializeAsync now writes Faulted with a sanitized reason (generation-checked) and host Stopped instead of latching Initializing with no LastError. m-6: SecurityClass-as-descriptor is documented at the Sql site (Commons field is a follow-up). m-7: a position-less JsonException renders '?' instead of blanks. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- .../SqlBrowseSession.cs | 5 +- .../ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs | 55 ++++++++++++++++-- .../SqlDriverFactoryExtensions.cs | 4 +- .../SqlPollReader.cs | 20 ++++++- .../SqlDriverTests.cs | 58 +++++++++++++++++++ .../SqlReinitConfigAdoptionTests.cs | 5 ++ 6 files changed, 137 insertions(+), 10 deletions(-) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs index 7feb6a0f..ceabf3a7 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Browser/SqlBrowseSession.cs @@ -197,7 +197,10 @@ internal sealed class SqlBrowseSession : IBrowseSession Name: column.Name, DriverDataType: _dialect.MapColumnType(column.DataType).ToString(), IsArray: false, - // The picker's descriptor line ("{type} · {SecurityClass}") — display only. Nullability is + // ⚠ SecurityClass is (ab)used as the picker's descriptor line ("{type} · {SecurityClass}"), + // exactly as the MQTT browser does. Every consumer today is display-only; a future one that + // parses it as SecurityClassification must not read the Sql value (review m-6 — a dedicated + // descriptor field on AttributeInfo is the Commons follow-up). Display only. Nullability is // what tells an operator a cell can come back NULL, which publishes Uncertain (or Bad under // nullIsBad), before they commit the tag. SecurityClass: column.IsNullable switch diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs index d7212245..ac10102c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriver.cs @@ -377,13 +377,30 @@ public sealed class SqlDriver // connection string as one unit; adopting options alone would poll a NEW tag set through the OLD // database, which is why this driver was excluded from the first pass. A null rebinder (direct // construction) or an empty/placeholder document keeps the constructor-supplied trio. - if (_rebind is not null && HasConfigBody(driverConfigJson)) + try { - var binding = _rebind(driverConfigJson); - ApplyBinding(binding.Options, binding.Dialect, binding.ConnectionString); - } + if (_rebind is not null && HasConfigBody(driverConfigJson)) + { + var binding = _rebind(driverConfigJson); + ApplyBinding(binding.Options, binding.Dialect, binding.ConnectionString); + } - BuildTagTable(); + BuildTagTable(); + } + catch (Exception ex) + { + // Review m-4: an unresolvable connectionStringRef or a malformed config threw here AFTER Initializing + // was written, and no poll may end Initializing — so the driver latched it with no LastError. Fault + // it like the liveness stage does: exception TYPE only on the operator surface (the factory's own + // messages are already sanitized, but this method cannot rely on that), full exception to the log. + var message = + $"Sql driver '{_driverInstanceId}' could not adopt its configuration: {ex.GetType().Name}. " + + "Check the connectionStringRef and the driver config JSON."; + _logger.LogError(ex, "Sql driver {DriverInstanceId} configuration could not be adopted.", _driverInstanceId); + if (WriteHealthIfCurrent(generation, new DriverHealth(DriverState.Faulted, ReadHealth().LastSuccessfulRead, message))) + TransitionHostTo(HostState.Stopped); + throw; + } try { @@ -539,12 +556,35 @@ public sealed class SqlDriver // The generation this read started under: if a Shutdown/Initialize lands while it is in flight, its // verdict describes a superseded configuration and must not touch the newer health (R06:S-9). var generation = Interlocked.Read(ref _lifecycleGeneration); + return await ReadThroughAsync(_reader, fullReferences, generation, cancellationToken).ConfigureAwait(false); + } + + /// + /// 's body against a specific reader — the one the caller captured. Internal so a + /// test can drive the "reader replaced between capture and use" race deterministically. + /// + internal async Task> ReadThroughAsync( + SqlPollReader reader, IReadOnlyList fullReferences, long generation, CancellationToken cancellationToken) + { try { - var result = await _reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); + var result = await reader.ReadDetailedAsync(fullReferences, cancellationToken).ConfigureAwait(false); ObservePollOutcome(result.Snapshots, result.AnsweredRefusalGroups, generation); return result.Snapshots; } + catch (SqlPollReaderDisposedException ex) when (!cancellationToken.IsCancellationRequested) + { + // A rebind swapped the reader under this poll (review m-1: a redeploy re-initialises without + // teardown, so the engine keeps polling). Retry ONCE on the current reader; if there is no newer + // one (the driver itself was disposed), hand the engine an exception it is told not to classify — + // it used to Degrade the driver as "poll failed: OperationCanceledException". + var current = Volatile.Read(ref _reader); + if (!ReferenceEquals(current, reader)) + return await ReadThroughAsync(current, fullReferences, generation, cancellationToken).ConfigureAwait(false); + + Volatile.Write(ref _classifiedByRead, ex); + throw; + } catch (OperationCanceledException) { // Teardown, not a database verdict — leave health and host state exactly as they were. @@ -1226,6 +1266,9 @@ public sealed class SqlDriver _reader.Dispose(); } + /// Re-runs on the current binding — a real reader swap, for the m-1 race test. + internal void RebindForTests() => ApplyBinding(_options, _dialect, _connectionString); + /// The live poll reader. Test observation of the S-3 lifecycle only. internal SqlPollReader ReaderForTests => _reader; diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs index 3bb727e5..d4f46fa8 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlDriverFactoryExtensions.cs @@ -199,8 +199,8 @@ public static class SqlDriverFactoryExtensions // connection-string literal is exactly the malformed text that lands here. This message becomes // InitializeFailed -> health LastError -> /hosts. Line/byte position carry no content, so they stay. throw new InvalidOperationException( - $"Sql driver config for '{driverInstanceId}' is not valid JSON (line {ex.LineNumber + 1}, byte " - + $"{ex.BytePositionInLine}). The parser's message is withheld because the malformed text may " + $"Sql driver config for '{driverInstanceId}' is not valid JSON (line " + + $"{(ex.LineNumber is { } line ? line + 1 : (object)"?")}, byte {ex.BytePositionInLine?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "?"}). The parser's message is withheld because the malformed text may " + "carry a connection string."); } } diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs index ffeb3804..0d3a96c5 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql/SqlPollReader.cs @@ -333,6 +333,10 @@ public sealed class SqlPollReader : IDisposable // leaves a sibling group running against a connection nobody is watching. var outcomes = await Task.WhenAll(groups).ConfigureAwait(false); + // Surfaced AFTER WhenAll, as one typed exception: a group throwing it would turn WhenAll into a plain + // TaskCanceledException and the caller could no longer tell "superseded reader" from a caller cancel. + if (outcomes.Any(o => ReferenceEquals(o, GroupResult.ReaderDisposed))) throw new SqlPollReaderDisposedException(); + var answeredRefusals = 0; for (var g = 0; g < plans.Count; g++) { @@ -382,7 +386,7 @@ public sealed class SqlPollReader : IDisposable { // A reader replaced by a reinitialize (and disposed) can still be holding a stale poll: it runs no // new group. Cancellation is the honest answer — the poll that asked has been superseded. - if (!TryAddRef()) throw new OperationCanceledException("The Sql poll reader was disposed."); + if (!TryAddRef()) return GroupResult.ReaderDisposed; try { @@ -477,6 +481,9 @@ public sealed class SqlPollReader : IDisposable { while (true) { + // A disposed reader starts no new group even while older ones are still in flight (review m-2): + // the gate would survive until they finish, but a replaced reader must not take new work. + if (Volatile.Read(ref _disposed) != 0) return false; var current = Volatile.Read(ref _refs); if (current == 0) return false; if (Interlocked.CompareExchange(ref _refs, current + 1, current) == current) return true; @@ -968,6 +975,10 @@ public sealed class SqlPollReader : IDisposable public static GroupResult Failed { get; } = new(SqlStatusCodes.BadCommunicationError, NoRows, NoColumns, []); + /// The reader was disposed before this group could start; surfaced as a typed exception. + public static GroupResult ReaderDisposed { get; } = + new(SqlStatusCodes.BadInternalError, NoRows, NoColumns, []); + /// /// The database answered with more rows than the plan can use — an ambiguous selector (R06:S-1). /// A configuration fact, not a database fault; the driver counts it as answer evidence. @@ -1003,6 +1014,13 @@ public sealed class SqlPollReader : IDisposable } } +/// +/// A poll reached a after it was disposed — a rebind replaced it, or the driver +/// was disposed. Not a database verdict: the driver retries on the current reader, and never degrades for it. +/// +public sealed class SqlPollReaderDisposedException() + : OperationCanceledException("The Sql poll reader was disposed (replaced by a rebind, or the driver was disposed)."); + /// One poll's snapshots plus the answer evidence they cannot carry themselves. /// One snapshot per requested reference, at the same index. /// diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs index 1eec94e5..ed93c7a5 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlDriverTests.cs @@ -484,6 +484,64 @@ public sealed class SqlDriverTests skipped.Reason.ShouldContain("Query"); } + /// + /// Review m-1. A redeploy re-initialises without teardown, so a poll can capture the reader just + /// before a rebind disposes it. That poll used to throw an OperationCanceledException the engine then + /// classified as a poll failure (Degrade). It now retries once on the current reader — and reads Good, + /// the positive control — with health untouched. + /// + [Fact] + public async Task APollOnAReplacedReader_retriesOnTheCurrentOne() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + var captured = driver.ReaderForTests; + + driver.RebindForTests(); + captured.GateDisposedForTests.ShouldBeTrue(); + + var snapshots = await driver.ReadThroughAsync( + captured, ["Speed"], driver.LifecycleGenerationForTests, CancellationToken.None); + + snapshots[0].Value.ShouldBe(SqlitePollFixture.PresentValue); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + } + + /// + /// With no newer reader (the driver's own reader was disposed) the disposed-reader exception still + /// propagates, but the engine's error callback must not degrade for it. Control: any other poll error + /// still degrades. + /// + [Fact] + public async Task ADisposedReaderError_isNotAPollFailure() + { + using var fixture = new SqlitePollFixture(); + await using var driver = NewDriver(fixture, KvEntry("Speed", SqlitePollFixture.PresentKey)); + await driver.InitializeAsync(ConfigJson, CancellationToken.None); + driver.ReaderForTests.Dispose(); + + // Awaited directly (as the poll engine does): an OperationCanceledException-derived throw ends the task + // Canceled, and only a direct await rethrows the original instance — which HandlePollError matches on. + Exception? caught = null; + try + { + await driver.ReadThroughAsync( + driver.ReaderForTests, ["Speed"], driver.LifecycleGenerationForTests, CancellationToken.None); + } + catch (Exception e) + { + caught = e; + } + + caught.ShouldBeOfType(); + driver.HandlePollError(caught!); + driver.GetHealth().State.ShouldBe(DriverState.Healthy); + + driver.HandlePollError(new InvalidOperationException("contract violation")); + driver.GetHealth().State.ShouldBe(DriverState.Degraded); + } + // ---- Driver.Sql-010: the health field is read-modify-written from several poll loops ---- [Fact] diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs index f9223f94..12e2fc22 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Sql.Tests/SqlReinitConfigAdoptionTests.cs @@ -81,6 +81,11 @@ public sealed class SqlReinitConfigAdoptionTests "old-server", customMessage: "a reinitialize that could not resolve its new connection must leave the previous binding whole"); + + // Review m-4: the failure is a fault with a reason, never a latched Initializing with no LastError. + var health = driver.GetHealth(); + health.State.ShouldBe(Core.Abstractions.DriverState.Faulted); + health.LastError.ShouldNotBeNull().ShouldContain("could not adopt its configuration"); } /// An empty/placeholder document keeps the constructor-supplied binding, so the many lifecycle From e7e84e05771dd457c4ae84fb6eefb107d96622b5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 22 Oct 2026 16:39:24 -0400 Subject: [PATCH 20/20] fix(mqtt): review minors - QoS 0 is not answer evidence, caller cancels stay cancels, rebirth wait cap, bound/trie pins (R06 G2 MQTT review minors 2 3 4 6 7 8) 2: only a PUBACKed (QoS >= 1) publish advances LastSuccessfulRead; a QoS 0 Sparkplug command completing means only that the bytes left the local socket. 3: a no-verdict SUBSCRIBE caused by the caller's own cancellation now throws OperationCanceledException (refs still Bad + pending), so the breaker no longer counts a pipeline timeout as a failure. 4: the rebirth wait is the window capped at 15 s, below the AdminUI 20 s per-call timeout. 6: a 10-topic test pins the 8-in-flight publish bound. 7: partial-level wildcards and $share filters are pinned never to confirm. 8: the intended init-vs-shutdown supersede is commented at the re-check. Refs #744 Claude-Session: https://claude.ai/code/session_01Ub9hGmYLQvn9XETzw7CMsT --- docs/drivers/Mqtt.md | 4 +- .../MqttBrowseSession.cs | 12 ++- .../MqttDriver.cs | 13 +++- .../MqttSubscriptionManager.cs | 13 ++++ .../MqttBirthObservationWindowTests.cs | 15 ++++ .../MqttDriverSessionLifecycleTests.cs | 73 +++++++++++++++++++ .../MqttSubscriptionManagerTests.cs | 28 ++++++- .../TopicFilterTrieTests.cs | 21 ++++++ 8 files changed, 171 insertions(+), 8 deletions(-) diff --git a/docs/drivers/Mqtt.md b/docs/drivers/Mqtt.md index d5c33835..5952b4fc 100644 --- a/docs/drivers/Mqtt.md +++ b/docs/drivers/Mqtt.md @@ -128,7 +128,7 @@ driver are pure organisational grouping in the RawPath. | `HostId` | `""` | Sparkplug Host Application identity — the `spBv1.0/STATE/{HostId}` topic. Optional for observation; **required** to claim. Set without `ActAsPrimaryHost`, the driver only observes | | `ActAsPrimaryHost` | `false` | Claim the Sparkplug Primary Host Application identity. **Primary-gated** — only the node holding the redundancy Primary role claims it. See [Primary host](#primary-host-actasprimaryhost) | | `RequestRebirthOnGap` | `true` | On a detected sequence-number gap, publish a rebirth-request NCMD instead of running on stale metric state | -| `BirthObservationWindowSeconds` | `15` | The Sparkplug browse picker's birth-collection window, clamped to 1–120 s (R06:M-4). The open budget grows by it; the open lingers while NBIRTH/DBIRTH traffic is still arriving (returning after a 1 s quiet spell — an idle plant costs 1 s); and an operator **Request rebirth** waits, bounded by it, for the births it asked for to arrive and go quiet for 500 ms, so the tree re-rendered afterwards carries them. A caller deadline during that wait still reports the publish. The runtime driver does not read it (its `DiscoverAsync` has no production caller — `SupportsOnlineDiscovery` is false) | +| `BirthObservationWindowSeconds` | `15` | The Sparkplug browse picker's birth-collection window, clamped to 1–120 s (R06:M-4). The open budget grows by it; the open lingers while NBIRTH/DBIRTH traffic is still arriving (returning after a 1 s quiet spell — an idle plant costs 1 s); and an operator **Request rebirth** waits, bounded by it and capped at 15 s (below the AdminUI's 20 s per-call timeout), for the births it asked for to arrive and go quiet for 500 ms, so the tree re-rendered afterwards carries them. A caller deadline during that wait still reports the publish. The runtime driver does not read it (its `DiscoverAsync` has no production caller — `SupportsOnlineDiscovery` is false) | `MqttDriverForm` authors all five in Sparkplug mode. The sub-object is **merged, never replaced** (a key a newer driver adds inside it survives an older AdminUI), and in `Plain` mode it is not touched at @@ -494,7 +494,7 @@ subscribe — including the device's own reconnect — so one stale write would **A batch publishes concurrently, one topic at a time (R06:M-8).** Writes to *different* topics in one `WriteAsync` go out together, at most 8 in flight, so a stalled broker costs a batch about one publish deadline rather than one per request; writes to the *same* topic stay sequential in request order, so the last one wins. -A successful publish (PUBACK, or a sent QoS 0 command) advances `LastSuccessfulRead` like an inbound message. +A PUBACKed (QoS 1) publish advances `LastSuccessfulRead` like an inbound message; a QoS 0 Sparkplug command does not, because its completion only means the bytes left the local socket. **Plain QoS 1 is at-least-once.** Know the consequences before pointing a Plain `writeTopic` at a non-idempotent device topic: diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs index b6bf8d9c..1b6b691c 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Browser/MqttBrowseSession.cs @@ -149,6 +149,13 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// private static readonly UTF8Encoding Utf8Strict = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + /// + /// Ceiling on the rebirth request's wait, below the AdminUI's 20 s BrowserSessionService.PerCallTimeout + /// (review minor 4): a configured window above it could never take effect there, and a rebirth aimed at an + /// offline edge node must not hold the operator's button for the whole window. + /// + internal static readonly TimeSpan MaxRebirthWait = TimeSpan.FromSeconds(15); + /// The quiet spell that ends a rebirth request's wait once its births have started arriving. private static readonly TimeSpan RebirthQuietInterval = TimeSpan.FromMilliseconds(500); @@ -225,6 +232,9 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession /// The Sparkplug birth-observation window this session was opened with (R06:M-4). internal TimeSpan BirthWindow { get; } + /// How long a rebirth request waits for its births: the window, capped at . + internal TimeSpan RebirthWait => BirthWindow < MaxRebirthWait ? BirthWindow : MaxRebirthWait; + /// How many NBIRTH/DBIRTH messages this session has observed — the settle loop's signal. internal long BirthCount => Interlocked.Read(ref _birthCount); @@ -987,7 +997,7 @@ internal sealed class MqttBrowseSession : IRebirthCapableBrowseSession try { await SettleBirthsAsync( - () => BirthCount, baseline, awaitFirstBirth: true, RebirthQuietInterval, BirthWindow, cancellationToken) + () => BirthCount, baseline, awaitFirstBirth: true, RebirthQuietInterval, RebirthWait, cancellationToken) .ConfigureAwait(false); } catch (OperationCanceledException) diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs index dec7569c..c265dce1 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttDriver.cs @@ -806,9 +806,11 @@ public sealed class MqttDriver .ConfigureAwait(false); results[index] = new WriteResult(0u); - // A PUBACK (or a sent QoS 0 command) is the broker answering — evidence the session works, - // so it feeds LastSuccessfulRead alongside inbound messages. - Interlocked.Exchange(ref _lastPublishUtcTicks, DateTime.UtcNow.Ticks); + // A PUBACK is the broker answering — evidence the session works, so it feeds + // LastSuccessfulRead alongside inbound messages. A completed QoS 0 publish is NOT: it only + // means the bytes reached the local socket buffer, and frequent Sparkplug commands would + // otherwise keep the watchdog's clock fresh over a half-open session (review minor 2). + if (encoded.Qos >= 1) Interlocked.Exchange(ref _lastPublishUtcTicks, DateTime.UtcNow.Ticks); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -1118,7 +1120,10 @@ public sealed class MqttDriver await StartPrimaryHostAsync(cancellationToken).ConfigureAwait(false); // Re-checked after publishing: a shutdown whose gate wait expired between the check above and - // here captured whatever it found and left these fields for us. We still hold the gate, so no + // here captured whatever it found and left these fields for us. Intended consequence (review + // minor 8): a perfectly healthy init that is just completing when a ShutdownAsync arrives also + // releases itself and throws OperationCanceledException — the queued shutdown would tear it down + // anyway, and the actor serialises init/shutdown, so only an abandoned init can meet this. We still hold the gate, so no // newer initialize can have published anything — everything in them is ours to release. if (!IsCurrentLifecycle(generation)) { diff --git a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs index 0c8b1a9a..2a80209d 100644 --- a/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs +++ b/src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt/MqttSubscriptionManager.cs @@ -454,6 +454,10 @@ public sealed class MqttSubscriptionManager if (outcome.TransportFailure is { } failure) { + // The refs are already Bad and pending either way. But a cancel of the CALLER's token (a pipeline + // timeout, host shutdown) is a cancellation, not a broker fault: surfacing it as + // InvalidOperationException made the resilience breaker count it as a failure (review minor 3). + var callerCancelled = cancellationToken.IsCancellationRequested; // No verdict at all (transport throw / SUBSCRIBE deadline): THROW, so the caller holds no handle // and DriverInstanceActor's health-tick reconcile re-subscribes. Returning a handle here was the // silent-death case — the actor thought the subscription live and never retried. The refs already @@ -464,6 +468,15 @@ public sealed class MqttSubscriptionManager _handleByRawPath.TryRemove(new KeyValuePair(def.Name, handle)); } + if (callerCancelled) + { + throw new OperationCanceledException( + $"MQTT driver '{_driverId}': SUBSCRIBE for {pending.Count} tag(s) was cancelled by the caller; " + + "the topics stay pending and will be retried.", + failure, + cancellationToken); + } + throw new InvalidOperationException( $"MQTT driver '{_driverId}': SUBSCRIBE for {pending.Count} tag(s) got no broker verdict " + $"({failure.GetType().Name}); the subscription will be retried.", diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs index b7f920af..3132d3b2 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttBirthObservationWindowTests.cs @@ -43,6 +43,21 @@ public sealed class MqttBirthObservationWindowTests public void BirthWindow_IsClamped(int configured, int expectedSeconds) => MqttDriverBrowser.BirthWindow(Sparkplug(configured)).ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + /// + /// Review minor 4: the rebirth wait is capped below the AdminUI's 20 s per-call timeout, so a long + /// configured window cannot hold the button past it (and a short one still governs). + /// + [Theory] + [InlineData(120, 15)] + [InlineData(15, 15)] + [InlineData(5, 5)] + public async Task RebirthWait_IsTheWindowCappedAt15Seconds(int windowSeconds, int expectedSeconds) + { + await using var s = new MqttBrowseSession( + MqttMode.SparkplugB, client: null, logger: null, birthWindow: TimeSpan.FromSeconds(windowSeconds)); + s.RebirthWait.ShouldBe(TimeSpan.FromSeconds(expectedSeconds)); + } + /// An idle plant costs one quiet interval, not the whole window. [Fact] public async Task Settle_WithNoBirths_ReturnsAfterOneQuietInterval() diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs index 14dbda13..8abf72aa 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttDriverSessionLifecycleTests.cs @@ -361,6 +361,79 @@ public sealed class MqttDriverSessionLifecycleTests driver.GetHealth().LastSuccessfulRead.ShouldNotBeNull(); } + /// + /// Review minor 6: the bound itself. Ten topics against a broker that never PUBACKs: exactly + /// PUBLISHes go out and the count then holds — a + /// regression to unbounded concurrency sends all ten. The stability check re-reads the count across + /// several condition polls rather than sleeping once. + /// + [Fact] + public async Task AWriteBatch_NeverHasMoreThanTheBoundInFlight() + { + using var broker = new MiniBroker(); // never PUBACKs + var options = BrokerOptions(broker.Port) with + { + ConnectTimeoutSeconds = 30, + RawTags = [.. Enumerable.Range(0, 10).Select(i => WritableTag($"Plant/Mqtt/dev1/Sp{i}", $"cmd/sp{i}"))], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + var write = driver.WriteAsync( + [.. Enumerable.Range(0, 10).Select(i => new WriteRequest($"Plant/Mqtt/dev1/Sp{i}", (double)i))], + cts.Token); + + await WaitForAsync(() => broker.Published.Count >= MqttDriver.MaxConcurrentPublishes, "the bound to fill"); + for (var i = 0; i < 10; i++) + { + await Task.Delay(50, TestContext.Current.CancellationToken); + broker.Published.Count.ShouldBe(MqttDriver.MaxConcurrentPublishes, "a slot freed with no PUBACK received"); + } + + await cts.CancelAsync(); + var results = await write.WaitAsync(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken); + results.Count.ShouldBe(10); + results.ShouldAllBe(r => r.StatusCode == 0x80050000u); + } + + /// + /// Review minor 2: a QoS 0 publish (a Sparkplug command) completing only means the bytes reached the + /// local socket buffer; the broker has not answered, so it must not advance LastSuccessfulRead. + /// The positive control is , where a QoS 1 + /// PUBACK does advance it. + /// + [Fact] + public async Task ASparkplugQos0Command_DoesNotAdvanceLastSuccessfulRead() + { + using var broker = new MiniBroker(); + var options = BrokerOptions(broker.Port) with + { + Mode = MqttMode.SparkplugB, + Sparkplug = new MqttSparkplugOptions { GroupId = "G", RequestRebirthOnGap = false }, + RawTags = + [ + new RawTagEntry( + "Plant/Mqtt/spb/Setpoint", + """{"groupId":"G","edgeNodeId":"E","metricName":"Setpoint","dataType":"Float64"}""", + WriteIdempotent: false), + ], + }; + await using var driver = new MqttDriver(options, "d", null); + await driver.InitializeAsync(Json(options), TestContext.Current.CancellationToken); + var before = driver.GetHealth().LastSuccessfulRead; + var publishedBefore = broker.Published.Count; + + var results = await driver.WriteAsync( + [new WriteRequest("Plant/Mqtt/spb/Setpoint", 4.5)], TestContext.Current.CancellationToken); + + results.ShouldHaveSingleItem().StatusCode.ShouldBe(0u); + await WaitForAsync( + () => broker.Published.Skip(publishedBefore).Any(p => p.Topic == "spBv1.0/G/NCMD/E"), + "the NCMD to reach the broker — the control that the write really went out"); + driver.GetHealth().LastSuccessfulRead.ShouldBe(before); + } + private static async Task WaitForAsync(Func condition, string what) { var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs index 906c6da4..3757c169 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/MqttSubscriptionManagerTests.cs @@ -553,7 +553,7 @@ public sealed class MqttSubscriptionManagerTests var raised = new List(); mgr.OnDataChange += (_, e) => raised.Add(e); - await Should.ThrowAsync(() => + await Should.ThrowAsync(() => mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken)); mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); @@ -561,6 +561,32 @@ public sealed class MqttSubscriptionManagerTests raised.ShouldContain(e => e.FullReference == OvenTempPath && e.Snapshot.StatusCode == BadCommunicationError); } + /// + /// Review minor 3: a cancel of the CALLER's token (a pipeline timeout, host shutdown) is a cancellation, + /// not a broker fault — it must surface as OperationCanceledException, or the resilience breaker counts + /// it as a failure. The refs are still Bad and pending, so the next reconnect retries them (the positive + /// control). The uncancelled no-verdict case throws InvalidOperationException + /// (). + /// + [Fact] + public async Task SubscribeAsync_CallerCancelled_ThrowsOperationCanceled_AndTheTopicStaysPending() + { + var transport = new FakeTransport { Throw = new OperationCanceledException() }; + var mgr = NewManager(transport); + mgr.Register([Tag(OvenTempPath, $$"""{"topic":"{{OvenTopic}}","payloadFormat":"Scalar","dataType":"Float64"}""")]); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + var ex = await Should.ThrowAsync(() => + mgr.SubscribeAsync([OvenTempPath], TimeSpan.FromSeconds(1), cts.Token)); + ex.ShouldNotBeOfType(); + mgr.Values.Read(OvenTempPath).StatusCode.ShouldBe(BadCommunicationError); + + transport.Throw = null; + await mgr.OnReconnectedAsync(TestContext.Current.CancellationToken); + transport.Calls.Last().Select(f => f.Topic).ShouldBe([OvenTopic]); + } + /// /// The reconcile path: once the actor re-subscribes after the throw, the topic IS re-SUBSCRIBEd and /// delivery resumes on the new handle — the positive control for the throw above. diff --git a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs index f21677e1..191b4596 100644 --- a/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs +++ b/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Mqtt.Tests/TopicFilterTrieTests.cs @@ -69,6 +69,27 @@ public sealed class TopicFilterTrieTests collect.Seen.Contains(1).ShouldBe(expected); } + /// + /// Review minor 7: partial-level wildcards and $share/… filters are outside the trie's level rules. + /// Whatever it yields, confirmation by the broker's comparer must produce no match — pinned so a later + /// "smarter" trie cannot start delivering them. + /// + [Theory] + [InlineData("a+", "ab")] + [InlineData("a/b#", "a/bc")] + [InlineData("a/b#", "a/b")] + [InlineData("$share/g/a/b", "a/b")] + [InlineData("$share/g/#", "a/b")] + public void Match_partialLevelWildcardsAndSharedFilters_neverConfirm(string filter, string topic) + { + var trie = TopicFilterTrie.Build([(filter, 1)]); + var collect = new Collect(); + trie.Match(topic, ref collect); + + collect.Seen.Where(_ => MqttTopicFilterComparer.Compare(topic, filter) == MqttTopicFilterCompareResult.IsMatch) + .ShouldBeEmpty(); + } + [Fact] public void Empty_MatchesNothing() {