fix(galaxy): stranded subscriptions after a no-replay reconnect go Bad + Degraded (#862)
With Reconnect.ReplayOnSessionLost=false a recreated gateway session carries none of the old advises, so every pre-reconnect subscription went silently deaf: nodes kept their last value as Good and health returned to Healthy (only a Warning log). ReplayAsync now records the stranded subscriptions and publishes BadNoCommunication (StatusCodeMap.BadNoCommunication, 0x80310000; value null — no driver value cache) for every tag of each, straight to OnDataChange so LastSuccessfulRead is not stamped. GetHealth reports Degraded naming the setting while any stranded subscription is still registered; it clears on UnsubscribeAsync (the host's re-subscribe path) or runtime teardown, and pruned ids never pin it. A subscribe after the reconnect is an ordinary new-session subscription. With nothing else flowing, the comms-loss watchdog re-initializes the driver after 90 s (a delayed replay). Tests: Runtime/StrandedSubscriptionTests (6). Docs: drivers/Galaxy.md, which also no longer claims the replay uses the gateway's ReplaySubscriptions RPC. Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
This commit is contained in:
+3
-2
@@ -63,8 +63,9 @@ public sealed class GalaxyDriverInfrastructureTests
|
||||
{
|
||||
// ReplayOnSessionLost was a dangling option — defined + documented but never
|
||||
// read. After the fix, setting it to false makes the reconnect replay path
|
||||
// skip SubscribeBulk (operator opts out of replay; the gateway's session-level
|
||||
// ReplaySubscriptions handles state restoration).
|
||||
// skip SubscribeBulk (operator opts out of replay). The recreated gateway session
|
||||
// carries none of the old advises, so those subscriptions are stranded — published
|
||||
// BadNoCommunication + driver Degraded (#862, StrandedSubscriptionTests).
|
||||
var sub = new ReplayCountingSubscriber();
|
||||
var opts = new GalaxyDriverOptions(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Config;
|
||||
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
|
||||
|
||||
/// <summary>
|
||||
/// #862 — with <c>Reconnect.ReplayOnSessionLost=false</c> a gateway reconnect keeps the driver's
|
||||
/// subscriptions but the recreated session carries none of their advises. That used to be invisible:
|
||||
/// every node kept its last value with Good quality and health went back to Healthy. Now each tag of
|
||||
/// each stranded subscription is published <c>BadNoCommunication</c> and the driver reports Degraded
|
||||
/// until those subscriptions are unsubscribed (or the runtime is torn down).
|
||||
/// </summary>
|
||||
public sealed class StrandedSubscriptionTests
|
||||
{
|
||||
private const uint BadNoCommunication = 0x80310000u;
|
||||
|
||||
private static GalaxyDriverOptions Opts(bool replay) => new(
|
||||
new GalaxyGatewayOptions("https://mxgw.test:5001", "key"),
|
||||
new GalaxyMxAccessOptions("OtOpcUa-A"),
|
||||
new GalaxyRepositoryOptions(WatchDeployEvents: false),
|
||||
new GalaxyReconnectOptions(ReplayOnSessionLost: replay));
|
||||
|
||||
private static MxEvent DataChange(int handle, double value) => new()
|
||||
{
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ItemHandle = handle,
|
||||
Value = new MxValue { DoubleValue = value },
|
||||
Quality = 192,
|
||||
SourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow),
|
||||
};
|
||||
|
||||
private static GalaxyDriver NewDriver(bool replay, HandleSubscriber subscriber) =>
|
||||
new("g", Opts(replay), hierarchySource: null, dataReader: null, dataWriter: null, subscriber: subscriber)
|
||||
{
|
||||
RecreateOverrideForTests = _ => Task.CompletedTask,
|
||||
};
|
||||
|
||||
private static async Task ReconnectAsync(GalaxyDriver driver)
|
||||
{
|
||||
await driver.InvokeReopenForTestAsync(TestContext.Current.CancellationToken);
|
||||
await driver.InvokeReplayForTestAsync(TestContext.Current.CancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The reconnect publishes BadNoCommunication for every tag of every stranded subscription, under
|
||||
/// that subscription's own handle, and degrades health with a message naming the setting.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Replay_disabled_reconnect_publishes_BadNoCommunication_and_degrades()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: false, subscriber);
|
||||
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => events.Enqueue(e);
|
||||
|
||||
var ab = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A", "Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
var c = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.C"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
|
||||
var callsBefore = Volatile.Read(ref subscriber.SubscribeCalls);
|
||||
var lastReadBefore = driver.GetHealth().LastSuccessfulRead;
|
||||
|
||||
await ReconnectAsync(driver);
|
||||
|
||||
Volatile.Read(ref subscriber.SubscribeCalls).ShouldBe(callsBefore, "replay is disabled — nothing is re-advised");
|
||||
var published = events.ToArray();
|
||||
published.Select(e => (e.FullReference, ((GalaxySubscriptionHandle)e.SubscriptionHandle).SubscriptionId))
|
||||
.OrderBy(x => x.FullReference)
|
||||
.ShouldBe([("Tank.A", ab.SubscriptionId), ("Tank.B", ab.SubscriptionId), ("Tank.C", c.SubscriptionId)]);
|
||||
published.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication && e.Snapshot.Value == null);
|
||||
|
||||
var health = driver.GetHealth();
|
||||
health.State.ShouldBe(DriverState.Degraded);
|
||||
health.LastError.ShouldNotBeNull().ShouldContain("ReplayOnSessionLost");
|
||||
health.LastError.ShouldContain("2 Galaxy subscription(s)");
|
||||
health.LastSuccessfulRead.ShouldBe(lastReadBefore, "a synthetic Bad publish is not evidence of a live gateway");
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(2);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Degraded clears exactly when the last stranded subscription is unsubscribed (what the host does on
|
||||
/// every re-subscribe), and the stranded handles' dead numbers are never sent to the new session.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task Degraded_clears_once_every_stranded_subscription_is_unsubscribed()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: false, subscriber);
|
||||
var a = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
var b = await driver.SubscribeAsync(["Tank.B"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
await ReconnectAsync(driver);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Degraded);
|
||||
|
||||
await driver.UnsubscribeAsync(a, TestContext.Current.CancellationToken);
|
||||
driver.GetHealth().State.ShouldBe(DriverState.Degraded, "Tank.B is still stranded");
|
||||
driver.GetHealth().LastError.ShouldNotBeNull().ShouldContain("1 Galaxy subscription(s)");
|
||||
|
||||
await driver.UnsubscribeAsync(b, TestContext.Current.CancellationToken);
|
||||
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
|
||||
subscriber.Unsubscribed.ShouldBeEmpty("old-session handle numbers must never reach the recreated session");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A subscribe after the reconnect is an ordinary new-session subscription: it receives data, is
|
||||
/// not counted as stranded, and keeps working once the stranded ones are gone.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task A_new_subscribe_after_the_reconnect_works_and_is_not_stranded()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: false, subscriber);
|
||||
var delivered = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => delivered.Enqueue(e);
|
||||
var old = await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
await ReconnectAsync(driver);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
|
||||
|
||||
var fresh = (GalaxySubscriptionHandle)await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(1, "the new subscription holds live new-session handles");
|
||||
var freshHandle = subscriber.LastHandle;
|
||||
|
||||
subscriber.Emit(DataChange(freshHandle, 7.5));
|
||||
await TestWait.UntilAsync(
|
||||
() => delivered.Any(e => e.Snapshot.StatusCode == 0 && (double?)e.Snapshot.Value == 7.5),
|
||||
"a subscribe after the reconnect must receive data");
|
||||
delivered.Last().SubscriptionHandle.ShouldBe(fresh);
|
||||
|
||||
await driver.UnsubscribeAsync(old, TestContext.Current.CancellationToken);
|
||||
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
|
||||
subscriber.Unsubscribed.ShouldBeEmpty();
|
||||
await driver.UnsubscribeAsync(fresh, TestContext.Current.CancellationToken);
|
||||
subscriber.Unsubscribed.ToArray().ShouldBe(new[] { freshHandle }, "the live subscription unadvises normally");
|
||||
}
|
||||
|
||||
/// <summary>Default replay re-advises everything: no Bad publish, no Degraded.</summary>
|
||||
[Fact]
|
||||
public async Task Replay_enabled_reconnect_publishes_no_Bad_and_stays_undegraded()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: true, subscriber);
|
||||
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => events.Enqueue(e);
|
||||
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
await ReconnectAsync(driver);
|
||||
|
||||
// Positive control: the replayed handle delivers, so the publish path was live while we looked.
|
||||
subscriber.Emit(DataChange(subscriber.LastHandle, 1.0));
|
||||
await TestWait.UntilAsync(() => !events.IsEmpty, "the replayed subscription must deliver");
|
||||
events.ShouldAllBe(e => e.Snapshot.StatusCode != BadNoCommunication);
|
||||
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
|
||||
}
|
||||
|
||||
/// <summary>A second reconnect re-asserts the stranded state (still one entry per subscription).</summary>
|
||||
[Fact]
|
||||
public async Task A_second_reconnect_restates_the_same_stranded_subscriptions()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: false, subscriber);
|
||||
var events = new ConcurrentQueue<DataChangeEventArgs>();
|
||||
driver.OnDataChange += (_, e) => events.Enqueue(e);
|
||||
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
|
||||
await ReconnectAsync(driver);
|
||||
await ReconnectAsync(driver);
|
||||
|
||||
events.Count.ShouldBe(2);
|
||||
events.ShouldAllBe(e => e.Snapshot.StatusCode == BadNoCommunication);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
|
||||
}
|
||||
|
||||
/// <summary>The runtime teardown forgets the stranded set along with the registry.</summary>
|
||||
[Fact]
|
||||
public async Task Shutdown_clears_the_stranded_state()
|
||||
{
|
||||
var subscriber = new HandleSubscriber();
|
||||
using var driver = NewDriver(replay: false, subscriber);
|
||||
await driver.SubscribeAsync(["Tank.A"], TimeSpan.Zero, TestContext.Current.CancellationToken);
|
||||
await ReconnectAsync(driver);
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(1);
|
||||
|
||||
await driver.ShutdownAsync(TestContext.Current.CancellationToken);
|
||||
|
||||
driver.StrandedSubscriptionCountForTests.ShouldBe(0);
|
||||
driver.GetHealth().State.ShouldNotBe(DriverState.Degraded);
|
||||
}
|
||||
|
||||
/// <summary>Hands out monotonic item handles, records unsubscribes, and streams emitted events.</summary>
|
||||
private sealed class HandleSubscriber : IGalaxySubscriber
|
||||
{
|
||||
private readonly Channel<MxEvent> _events = Channel.CreateUnbounded<MxEvent>();
|
||||
private int _nextHandle = 100;
|
||||
|
||||
/// <summary>Number of SubscribeBulk calls.</summary>
|
||||
public int SubscribeCalls;
|
||||
|
||||
/// <summary>Gets the most recently issued item handle.</summary>
|
||||
public int LastHandle => Volatile.Read(ref _nextHandle);
|
||||
|
||||
/// <summary>Gets every item handle passed to UnsubscribeBulk.</summary>
|
||||
public ConcurrentQueue<int> Unsubscribed { get; } = new();
|
||||
|
||||
/// <summary>Pushes an event onto the stream.</summary>
|
||||
/// <param name="ev">The event.</param>
|
||||
public void Emit(MxEvent ev) => _events.Writer.TryWrite(ev).ShouldBeTrue();
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<SubscribeResult>> SubscribeBulkAsync(
|
||||
IReadOnlyList<string> fullReferences, int bufferedUpdateIntervalMs, CancellationToken cancellationToken)
|
||||
{
|
||||
Interlocked.Increment(ref SubscribeCalls);
|
||||
IReadOnlyList<SubscribeResult> results = [.. fullReferences.Select(r => new SubscribeResult
|
||||
{
|
||||
TagAddress = r,
|
||||
ItemHandle = Interlocked.Increment(ref _nextHandle),
|
||||
WasSuccessful = true,
|
||||
})];
|
||||
return Task.FromResult(results);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UnsubscribeBulkAsync(IReadOnlyList<int> itemHandles, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var h in itemHandles) Unsubscribed.Enqueue(h);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync([EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await foreach (var ev in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
yield return ev;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user