Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests/Runtime/GatewayGalaxyAlarmFeedTests.cs
T
Joseph Doherty 4fdc3708f4 test(galaxy): presence waits survive a starved/loaded runner; absence asserted behind positive controls (#858)
Under heavy parallel load (14 concurrent test processes + 14 CPU burners, with
DOTNET_ThreadPool_ForceMinWorkerThreads=1 as the reproducer) the Galaxy tests
with 1-3 s fixed waits failed 16/70 runs. Every failure was a background loop
started with Task.Run (alarm feed, DeployWatcher, EventPump, ReconnectSupervisor)
not being SCHEDULED within the ceiling - e.g. DeployWatcher's stream had not even
been opened after 2 s. No production race: the code under test was checked for
each (alarm feed dispatch is synchronous in-order, DeployWatcher/ReconnectSupervisor
state is lock- or loop-local, subscribe registers before returning).

- New shared TestWait (UntilAsync / ForAsync / HealthyAsync): 30 s presence
  ceiling, failure reports elapsed ms + an observed-state snapshot (#856 shape;
  GalaxyDriverFixRound1Tests.WaitUntilAsync now delegates to it).
- Converted the short waits in GatewayGalaxyAlarmFeedTests, DeployWatcherTests,
  GalaxyDriverSubscribeTests, GalaxyDriverRawPathKeyingTests (its WaitForAsync
  never threw on timeout), EventPumpStreamFaultTests, EventPumpReplayGapTests,
  ReconnectSupervisorTests; FixRound2 InitBudget 2 s -> 10 s (the parked fetch
  never completes, so any finite bound keeps the verdict).
- Absence by short sleep replaced with positive controls: DeployWatcher bootstrap
  and same-deploy-time (follow with a real change, assert exactly that one raise -
  verified red when suppression is removed), subscribe failed-tag / unsubscribed
  handle (follow with a live handle's event), faulted pump (wait for its
  onStreamFault instead of 100 ms).
- Fixed sleeps that raced scheduling: subscribe-once read tests wait for the
  read's registration (TrackedSubscriptionCountForTests) instead of 50 ms;
  supervisor Dispose test waits for reopen entry instead of 50 ms; the
  already-Healthy WaitForHealthy test asserts synchronous completion instead of a
  wall-clock bound. FakeSubscriber.Map is now a ConcurrentDictionary.

Loaded run (14 parallel x 5, 14 burners, ForceMinWorkerThreads=1):
before 16/70 failed runs; after 0/350 (70 + 210 + 70). Without the thread-pool
override: 0/70 before and after on this host. Serial: 472 passed, 8 skipped.

Claude-Session: https://claude.ai/code/session_01GT67AD99zQQqpLCVm3p7xk
2026-11-11 10:18:22 -05:00

266 lines
11 KiB
C#

using System.Runtime.CompilerServices;
using Google.Protobuf.WellKnownTypes;
using ZB.MOM.WW.MxGateway.Contracts.Proto;
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Runtime;
// Core.Abstractions now also defines an AlarmTransitionKind (the AlarmEventArgs surface
// kind). This file drives the gateway proto, so pin the bare token to the proto enum to
// preserve the existing references unchanged.
using AlarmTransitionKind = ZB.MOM.WW.MxGateway.Contracts.Proto.AlarmTransitionKind;
namespace ZB.MOM.WW.OtOpcUa.Driver.Galaxy.Tests.Runtime;
/// <summary>
/// Pins <see cref="GatewayGalaxyAlarmFeed"/> — the session-less consumer of the
/// gateway's <c>StreamAlarms</c> feed. Synthetic <see cref="AlarmFeedMessage"/>s go
/// in through the stream-factory seam; the feed fires <c>OnAlarmTransition</c> with
/// decoded payloads and mapped severity buckets, drops malformed messages, and
/// re-opens the stream after a transport fault.
/// </summary>
public sealed class GatewayGalaxyAlarmFeedTests
{
/// <summary>Verifies that the feed decodes active alarm snapshots and live transitions.</summary>
[Fact]
public async Task Decodes_active_alarm_snapshot_then_live_transition()
{
var raise = new DateTime(2026, 5, 1, 12, 0, 0, DateTimeKind.Utc);
var messages = new[]
{
SnapshotMessage("Tank01.Level.HiHi", AlarmConditionState.Active, severity: 750,
lastTransition: raise),
SnapshotMessage("Tank02.Level.HiHi", AlarmConditionState.ActiveAcked, severity: 500,
lastTransition: raise, operatorUser: "alice", operatorComment: "investigating"),
new AlarmFeedMessage { SnapshotComplete = true },
TransitionMessage("Tank01.Level.HiHi", AlarmTransitionKind.Clear, severity: 750,
transitionTime: raise.AddMinutes(5), originalRaise: raise),
};
var observed = new List<GalaxyAlarmTransition>();
var got3 = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var feed = new GatewayGalaxyAlarmFeed(
(_, ct) => OpenStream(messages, ct), clientName: "FeedTest");
feed.OnAlarmTransition += (_, t) =>
{
lock (observed)
{
observed.Add(t);
if (observed.Count == 3) got3.TrySetResult(true);
}
};
feed.Start();
await TestWait.ForAsync(got3.Task, "snapshot + transition should dispatch",
() => { lock (observed) return $"dispatched={observed.Count}/3"; });
observed.Count.ShouldBe(3);
// Active (unacked) snapshot entry → Retrigger (#647 review: forces the upstream's unack on the host).
observed[0].AlarmFullReference.ShouldBe("Tank01.Level.HiHi");
observed[0].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Retrigger);
observed[0].SeverityBucket.ShouldBe(AlarmSeverity.Critical);
observed[0].RawMxAccessSeverity.ShouldBe(750);
// Acknowledged snapshot entry → ActiveAcknowledged (#656 — an Acknowledge kept the host's prior Active, so a
// cold host projected it INACTIVE), operator fields preserved.
observed[1].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.ActiveAcknowledged);
observed[1].OperatorUser.ShouldBe("alice");
observed[1].OperatorComment.ShouldBe("investigating");
// #656: snapshot entries re-state upstream state; the live transition is unflagged.
observed[0].IsRestatement.ShouldBeTrue();
observed[1].IsRestatement.ShouldBeTrue();
// Live transition after snapshot_complete.
observed[2].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Clear);
observed[2].OriginalRaiseTimestampUtc.ShouldBe(raise);
observed[2].IsRestatement.ShouldBeFalse();
}
/// <summary>
/// Gitea #656 — an Inactive snapshot entry is a re-stated Clear; a LIVE Retrigger (a genuine MXAccess
/// re-occurrence) is never flagged, so the host keeps its history row.
/// </summary>
[Fact]
public async Task Inactive_snapshot_entry_is_a_restated_clear_and_a_live_retrigger_is_unflagged()
{
var at = new DateTime(2026, 5, 1, 12, 0, 0, DateTimeKind.Utc);
var messages = new[]
{
SnapshotMessage("Tank01.Level.HiHi", AlarmConditionState.Inactive, severity: 750, lastTransition: at),
new AlarmFeedMessage { SnapshotComplete = true },
TransitionMessage("Tank01.Level.HiHi", AlarmTransitionKind.Retrigger, severity: 750,
transitionTime: at.AddMinutes(1)),
};
var observed = new List<GalaxyAlarmTransition>();
var got2 = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var feed = new GatewayGalaxyAlarmFeed((_, ct) => OpenStream(messages, ct), clientName: "FeedTest656");
feed.OnAlarmTransition += (_, t) =>
{
lock (observed)
{
observed.Add(t);
if (observed.Count == 2) got2.TrySetResult(true);
}
};
feed.Start();
await TestWait.ForAsync(got2.Task, "restated snapshot Clear + live Retrigger should dispatch",
() => { lock (observed) return $"dispatched={observed.Count}/2"; });
observed[0].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Clear);
observed[0].IsRestatement.ShouldBeTrue();
observed[1].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Retrigger);
observed[1].IsRestatement.ShouldBeFalse();
}
/// <summary>Verifies that the feed drops transitions with unspecified kind and empty messages.</summary>
[Fact]
public async Task Drops_transition_with_unspecified_kind_and_empty_message()
{
var messages = new[]
{
TransitionMessage("Tank01.Level.HiHi", AlarmTransitionKind.Unspecified, severity: 100,
transitionTime: DateTime.UtcNow),
new AlarmFeedMessage(), // empty oneof — version skew
TransitionMessage("Tank01.Level.HiHi", AlarmTransitionKind.Raise, severity: 600,
transitionTime: DateTime.UtcNow),
};
var observed = new List<GalaxyAlarmTransition>();
var gotOne = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var feed = new GatewayGalaxyAlarmFeed(
(_, ct) => OpenStream(messages, ct), clientName: "FeedTest");
feed.OnAlarmTransition += (_, t) =>
{
lock (observed)
{
observed.Add(t);
gotOne.TrySetResult(true);
}
};
feed.Start();
await TestWait.ForAsync(gotOne.Task, "the well-formed Raise should dispatch",
() => { lock (observed) return $"dispatched={observed.Count}"; });
// Only the well-formed Raise survives; the Unspecified + empty messages drop.
observed.ShouldHaveSingleItem();
observed[0].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Raise);
observed[0].SeverityBucket.ShouldBe(AlarmSeverity.High);
}
/// <summary>Verifies that the feed reopens the stream after a transport fault.</summary>
[Fact]
public async Task Reopens_stream_after_a_transport_fault()
{
var calls = 0;
var liveTransition = new[]
{
TransitionMessage("Tank01.Level.HiHi", AlarmTransitionKind.Raise, severity: 750,
transitionTime: DateTime.UtcNow),
};
var observed = new List<GalaxyAlarmTransition>();
var gotOne = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
await using var feed = new GatewayGalaxyAlarmFeed(
(_, ct) =>
{
// First open faults; the feed must reconnect and succeed on the retry.
if (Interlocked.Increment(ref calls) == 1)
{
throw new InvalidOperationException("synthetic stream fault");
}
return OpenStream(liveTransition, ct);
},
clientName: "ReconnectTest",
reconnectDelay: TimeSpan.FromMilliseconds(20));
feed.OnAlarmTransition += (_, t) =>
{
observed.Add(t);
gotOne.TrySetResult(true);
};
feed.Start();
await TestWait.ForAsync(gotOne.Task, "the feed should reopen the stream and deliver after a fault",
() => $"streamOpens={Volatile.Read(ref calls)}");
calls.ShouldBeGreaterThanOrEqualTo(2);
observed.ShouldHaveSingleItem();
observed[0].TransitionKind.ShouldBe(GalaxyAlarmTransitionKind.Raise);
}
/// <summary>
/// Yields each message in order, then holds the stream open until the feed is
/// disposed — mirrors a live server-streaming RPC that does not complete on its
/// own.
/// </summary>
private static async IAsyncEnumerable<AlarmFeedMessage> OpenStream(
IEnumerable<AlarmFeedMessage> messages,
[EnumeratorCancellation] CancellationToken ct = default)
{
foreach (var message in messages)
{
ct.ThrowIfCancellationRequested();
yield return message;
await Task.Yield();
}
await Task.Delay(Timeout.Infinite, ct);
}
private static AlarmFeedMessage SnapshotMessage(
string fullReference,
AlarmConditionState state,
int severity,
DateTime lastTransition,
string operatorUser = "",
string operatorComment = "")
=> new()
{
ActiveAlarm = new ActiveAlarmSnapshot
{
AlarmFullReference = fullReference,
SourceObjectReference = fullReference.Split('.')[0],
AlarmTypeName = "AnalogLimitAlarm.HiHi",
Severity = severity,
CurrentState = state,
Category = "Process",
Description = "Tank high-high level",
LastTransitionTimestamp = Timestamp.FromDateTime(lastTransition),
OperatorUser = operatorUser,
OperatorComment = operatorComment,
},
};
private static AlarmFeedMessage TransitionMessage(
string fullReference,
AlarmTransitionKind kind,
int severity,
DateTime transitionTime,
DateTime? originalRaise = null)
{
var body = new OnAlarmTransitionEvent
{
AlarmFullReference = fullReference,
SourceObjectReference = fullReference.Split('.')[0],
AlarmTypeName = "AnalogLimitAlarm.HiHi",
TransitionKind = kind,
Severity = severity,
TransitionTimestamp = Timestamp.FromDateTime(transitionTime),
Category = "Process",
Description = "Tank high-high level",
};
if (originalRaise is { } orts)
{
body.OriginalRaiseTimestamp = Timestamp.FromDateTime(orts);
}
return new AlarmFeedMessage { Transition = body };
}
}