diff --git a/docs/requirements/Component-DataConnectionLayer.md b/docs/requirements/Component-DataConnectionLayer.md index 2238cacb..3cd9ebc8 100644 --- a/docs/requirements/Component-DataConnectionLayer.md +++ b/docs/requirements/Component-DataConnectionLayer.md @@ -80,7 +80,7 @@ All protocols produce the same value tuple consumed by Instance Actors. Before t - Session-based: `OpenSession` + `Register` on connect; `AddItem` + `Advise` per subscription; value changes arrive on the gateway's server-streaming event feed (`StreamEvents`), resumable via `worker_sequence`. - Read/Write via `ReadBulk` / `WriteBulk`; writes carry a configurable `WriteUserId`. Quality maps the OPC-style quality byte (≥192 Good, ≥64 Uncertain, else Bad), with a failing MXAccess status proxy treated as Bad. - Galaxy hierarchy browse via the separate `GalaxyRepositoryClient` — objects are navigable nodes (keyed by Galaxy gobject id), attributes are selectable leaves (keyed by full tag reference). Browse is **lazy and attribute-light**: navigation uses `BrowseChildren` with `include_attributes=false` (child objects only), and an object's own attributes are fetched only when it is expanded, via `DiscoverHierarchy(root=, max_depth=0)` scoped to that single object. This keeps each browse level's reply small; inlining every child's full attribute set could exceed the Akka remote frame and silently drop the reply. -- Disconnect detection: a fault on the event stream raises `IDataConnection.Disconnected`, driving the same reconnection state machine as OPC UA. +- Disconnect detection: a fault on the event stream raises `IDataConnection.Disconnected`, driving the same reconnection state machine as OPC UA. **A graceful (OK-status) end of the event stream while the adapter did not cancel it is ALSO a disconnect** (fix 2026-08-20): the gateway completes a session's subscriber channels with no error when the session is closed from its dashboard or on a graceful gateway shutdown (`CompleteAllSubscribers(error: null)`), so before the fix `RunEventLoopAsync` simply returned, the adapter stayed `Connected`, subscribed values froze at their last GOOD quality and every write failed with `Session … was not found` until a process restart (QA incident, vq03 Z28061 MoveIn, one hour of silent loss; the 10-second `NotFound` heartbeat in the logs was the unresolved-tag resolution probe re-subscribing on the dead session). Same bug class as the gRPC site-stream client's graceful-completion reconnect. Pinned by `MxGatewayDataConnectionReconnectTests.EventStreamGracefulCompletion_*` (signals once while connected; silent for a superseded loop and after our own `DisconnectAsync`). Follow-up for the gateway SDK: `NotFound` maps to a generic `MxGatewayException` with only the detail text, so a write/subscribe hitting a dead session cannot be classified as a session loss without string matching — a typed session-not-found exception would allow a second, reactive detection layer. - Implemented as `MxGatewayDataConnection` over an `IMxGatewayClient` seam; the seam is decoupled from the generated gRPC types (only `RealMxGatewayClient` references them), so the adapter is fully unit-testable with a fake. ## Endpoint Redundancy diff --git a/src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Adapters/MxGatewayDataConnection.cs b/src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Adapters/MxGatewayDataConnection.cs index c4d929fe..a4aaf31e 100644 --- a/src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Adapters/MxGatewayDataConnection.cs +++ b/src/ZB.MOM.WW.ScadaBridge.DataConnectionLayer/Adapters/MxGatewayDataConnection.cs @@ -182,6 +182,23 @@ public class MxGatewayDataConnection if (_tagToSub.TryGetValue(update.TagPath, out var subId) && _subs.TryGetValue(subId, out var s)) s.Callback(update.TagPath, new TagValue(update.Value, update.Quality, update.Timestamp)); }, ct); + + // A GRACEFUL end of the event stream (OK status, no fault) while we did not ask + // for it means the gateway closed the session underneath us — a dashboard + // "close session", or a graceful gateway shutdown, both complete the per-session + // subscriber channels with no error (CompleteAllSubscribers(error: null)). + // Before this branch existed the loop simply returned: the adapter stayed + // Connected, every subscribed value froze at its last GOOD quality, and every + // write hit the dead session ("Session … was not found") until a process + // restart — QA incident 2026-08-20, vq03 Z28061 MoveIn, one hour of silent loss. + // Same bug class as SiteStreamGrpcClient's graceful-completion reconnect + // (commit 34a3f4bb). A cancelled token is our own teardown, never a disconnect. + if (!ct.IsCancellationRequested) + { + _logger.LogWarning( + "MxGateway event stream ended without a fault while connected — the gateway closed the session; signalling disconnect"); + RaiseDisconnected(); + } } catch (OperationCanceledException) { diff --git a/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/MxGatewayDataConnectionReconnectTests.cs b/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/MxGatewayDataConnectionReconnectTests.cs index b50cbfd3..c125963c 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/MxGatewayDataConnectionReconnectTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/MxGatewayDataConnectionReconnectTests.cs @@ -1,6 +1,8 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; using Grpc.Core; using Microsoft.Extensions.Logging.Abstractions; using NSubstitute; + using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters; namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters; @@ -93,6 +95,67 @@ public class MxGatewayDataConnectionReconnectTests Assert.Equal(0, Volatile.Read(ref disconnectsRef.Value)); } + + /// + /// QA incident 2026-08-20 (vq03 Z28061 MoveIn): an operator closed ScadaBridge's session + /// from the MxAccess Gateway dashboard. The gateway completes the session's event stream + /// GRACEFULLY (CompleteAllSubscribers(error: null), OK status) — the same path a + /// graceful gateway shutdown takes — so RunEventLoopAsync returned normally and the + /// adapter never raised Disconnected: values froze at their last GOOD quality and + /// every write went to the dead session (Session … was not found) for an hour, until + /// a process restart. A non-cancelled end of the event stream IS a disconnect. + /// + [Fact] + public async Task EventStreamGracefulCompletion_WhileConnected_SignalsDisconnected() + { + var loops = new List(); + var loopTokens = new List(); + var adapter = CreateAdapterWithControllableLoops(loops, loopTokens, out var disconnectsRef); + var details = new Dictionary { ["Endpoint"] = "http://gw:5000", ["ApiKey"] = "k" }; + + await adapter.ConnectAsync(details); + await WaitUntilAsync(() => loopTokens.Count == 1); + + loops[0].SetResult(); // server ended the stream with OK — session closed underneath us + await WaitUntilAsync(() => Volatile.Read(ref disconnectsRef.Value) == 1); + Assert.Equal(ConnectionHealth.Disconnected, adapter.Status); + } + + [Fact] + public async Task EventStreamGracefulCompletion_OfSupersededLoop_DoesNotSignalDisconnected() + { + var loops = new List(); + var loopTokens = new List(); + var adapter = CreateAdapterWithControllableLoops(loops, loopTokens, out var disconnectsRef); + var details = new Dictionary { ["Endpoint"] = "http://gw:5000", ["ApiKey"] = "k" }; + + await adapter.ConnectAsync(details); + await WaitUntilAsync(() => loopTokens.Count == 1); + await adapter.ConnectAsync(details); // reconnect: loop #0 is cancelled + await WaitUntilAsync(() => loopTokens.Count == 2); + + loops[0].SetResult(); // the OLD loop winds down normally after its cancel + await Task.Delay(200); + Assert.Equal(0, Volatile.Read(ref disconnectsRef.Value)); // must not flap the new session + } + + [Fact] + public async Task EventStreamGracefulCompletion_AfterDisconnectAsync_DoesNotSignalDisconnected() + { + var loops = new List(); + var loopTokens = new List(); + var adapter = CreateAdapterWithControllableLoops(loops, loopTokens, out var disconnectsRef); + var details = new Dictionary { ["Endpoint"] = "http://gw:5000", ["ApiKey"] = "k" }; + + await adapter.ConnectAsync(details); + await WaitUntilAsync(() => loopTokens.Count == 1); + await adapter.DisconnectAsync(); // our own teardown cancels the loop + + loops[0].SetResult(); + await Task.Delay(200); + Assert.Equal(0, Volatile.Read(ref disconnectsRef.Value)); + } + private sealed class IntRef { public int Value; } private static MxGatewayDataConnection CreateAdapterWithControllableLoops(