dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve all high-severity architecture review findings

Browse Source
This commit is contained in:
Joseph Doherty
2026-09-09 16:13:58 -04:00
parent 8bfe65512e
commit 146a13238f
192 changed files with 3160 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Actors/HighReviewRegressionTests.cs
+247
View File
@@ -0,0 +1,247 @@
using Akka.Actor;
using Akka.TestKit.Xunit2;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Protocol;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Actors;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Actors;
public class HighReviewRegressionTests : TestKit
{
private readonly ISiteHealthCollector _health = Substitute.For<ISiteHealthCollector>();
private static IDataConnection Adapter()
{
var adapter = Substitute.For<IDataConnection, IAlarmSubscribableConnection>();
adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>()).Returns(Task.CompletedTask);
adapter.SubscribeAsync(Arg.Any<string>(), Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>()).Returns("handle");
adapter.ReadAsync(Arg.Any<string>(), Arg.Any<CancellationToken>()).Returns(new ReadResult(false, null, null));
return adapter;
}
private IActorRef Actor(IDataConnection adapter) => Sys.ActorOf(Props.Create(() => new DataConnectionActor(
"conn", adapter, new DataConnectionOptions { ReconnectInterval = TimeSpan.FromMilliseconds(50), TagResolutionRetryInterval = TimeSpan.FromHours(1), SeedReadMaxAttempts = 1 },
_health, Substitute.For<IDataConnectionFactory>(), "OpcUa")));
private void Connected() => AwaitCondition(() => _health.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "UpdateConnectionHealth" && c.GetArguments()[1] is ConnectionHealth.Connected));
private static SubscribeTagsRequest Subscribe(string correlation = "s") => new(correlation, "instance", "conn", ["tag"], DateTimeOffset.UtcNow);
private DataConnectionHealthReport Health(IActorRef actor)
{
actor.Tell(new DataConnectionActor.GetHealthReport());
return ExpectMsg<DataConnectionHealthReport>();
}
[Fact]
public void ResolutionTick_DoesNotDuplicateAnInitialSubscribeAlreadyInFlight()
{
var adapter = new FakeBatchDataConnection();
adapter.FailingTags.Add("tag");
var actor = Actor(adapter);
var first = CreateTestProbe();
actor.Tell(Subscribe(), first.Ref);
first.ExpectMsg<TagValueUpdate>();
first.ExpectMsg<SubscribeTagsResponse>();
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
adapter.SubscribeGate = gate.Task;
adapter.FailingTags.Clear();
var second = CreateTestProbe();
actor.Tell(Subscribe("second") with { InstanceUniqueName = "second-instance" }, second.Ref);
AwaitCondition(() => adapter.SubscribeBatches.Count == 2);
var unexpected = CreateTestProbe();
adapter.BatchSubscribeThrows = () => { unexpected.Ref.Tell("duplicate subscribe"); return null; };
actor.Tell(new DataConnectionActor.RetryTagResolution());
try { unexpected.ExpectNoMsg(TimeSpan.FromMilliseconds(200)); }
finally { gate.SetResult(); }
}
[Fact]
public void OriginalOwnerDies_InFlightHandleStillServesDifferentInstance()
{
var adapter = Adapter();
var gate = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
adapter.SubscribeAsync("tag", Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>()).Returns(gate.Task);
var actor = Actor(adapter);
var a = CreateTestProbe(); var b = CreateTestProbe();
actor.Tell(Subscribe(), a.Ref);
AwaitCondition(() => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "SubscribeAsync"));
actor.Tell(Subscribe("b") with { InstanceUniqueName = "other-instance" }, b.Ref);
b.ExpectMsg<SubscribeTagsResponse>();
Health(actor); // mailbox barrier after B's registration
var beforeDeath = _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "UpdateTagResolution");
Sys.Stop(a.Ref);
AwaitCondition(() => _health.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "UpdateTagResolution") > beforeDeath);
// The remaining instance still needs the tag, even though A started the I/O.
Assert.Equal(1, Health(actor).TotalSubscribedTags);
gate.SetResult("shared-handle");
AwaitAssert(() => Assert.Equal(1, Health(actor).ResolvedTags), TimeSpan.FromSeconds(1));
Assert.DoesNotContain(adapter.ReceivedCalls(), c => c.GetMethodInfo().Name == "UnsubscribeAsync");
}
[Fact]
public void SubscriberTermination_DuringSubscribe_ReleasesLateHandleWithoutResurrection()
{
var adapter = Adapter();
var gate = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
adapter.SubscribeAsync("tag", Arg.Any<SubscriptionCallback>(), Arg.Any<CancellationToken>()).Returns(gate.Task);
var actor = Actor(adapter);
var subscriber = CreateTestProbe();
actor.Tell(Subscribe(), subscriber.Ref);
AwaitCondition(() => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "SubscribeAsync"));
Assert.Equal(1, Health(actor).TotalSubscribedTags);
Sys.Stop(subscriber.Ref);
AwaitAssert(() => Assert.Equal(0, Health(actor).TotalSubscribedTags));
gate.SetResult("late-handle");
AwaitCondition(() => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "UnsubscribeAsync" && Equals(c.GetArguments()[0], "late-handle")));
Assert.Equal(0, Health(actor).ResolvedTags);
}
[Fact]
public void Replacement_DuringSubscribe_AdoptsOnlyStillRequestedTags()
{
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var adapter = new FakeBatchDataConnection { SubscribeGate = gate.Task };
var actor = Actor(adapter);
var old = CreateTestProbe(); var current = CreateTestProbe();
actor.Tell(new SubscribeTagsRequest("old", "instance", "conn", ["keep", "drop"], DateTimeOffset.UtcNow), old.Ref);
AwaitCondition(() => adapter.SubscribeBatches.Count == 1);
actor.Tell(new SubscribeTagsRequest("new", "instance", "conn", ["keep"], DateTimeOffset.UtcNow), current.Ref);
current.ExpectMsg<SubscribeTagsResponse>();
Assert.Equal(1, Health(actor).TotalSubscribedTags);
gate.SetResult();
AwaitCondition(() => adapter.UnsubscribeBatches.SelectMany(b => b).Contains("sub-2"));
AwaitAssert(() => Assert.Equal(1, Health(actor).ResolvedTags));
Assert.DoesNotContain("sub-1", adapter.UnsubscribeBatches.SelectMany(b => b));
Sys.Stop(old.Ref);
Assert.Equal(1, Health(actor).TotalSubscribedTags);
}
[Fact]
public void SubscriberTermination_DuringResolution_ReleasesLateHandle()
{
var adapter = new FakeBatchDataConnection();
adapter.FailingTags.Add("tag");
var options = new DataConnectionOptions { TagResolutionRetryInterval = TimeSpan.FromMilliseconds(500), SeedReadMaxAttempts = 1 };
var actor = Sys.ActorOf(Props.Create(() => new DataConnectionActor("conn", adapter, options, _health, Substitute.For<IDataConnectionFactory>(), "OpcUa")));
var subscriber = CreateTestProbe();
actor.Tell(Subscribe(), subscriber.Ref);
subscriber.ExpectMsg<TagValueUpdate>();
subscriber.ExpectMsg<SubscribeTagsResponse>();
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
adapter.SubscribeGate = gate.Task;
adapter.FailingTags.Clear();
AwaitCondition(() => adapter.SubscribeBatches.Count == 2);
Sys.Stop(subscriber.Ref);
AwaitAssert(() => Assert.Equal(0, Health(actor).TotalSubscribedTags));
gate.SetResult();
AwaitCondition(() => adapter.UnsubscribeBatches.SelectMany(b => b).Any());
Assert.Equal(0, Health(actor).ResolvedTags);
}
[Fact]
public void DisconnectedSubscribeRetries_KeepOnlyLatestRequest()
{
var adapter = Adapter();
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>()).Returns(gate.Task);
var actor = Actor(adapter);
for (var i = 0; i < 100; i++) actor.Tell(Subscribe(i.ToString()));
Health(actor); // all retries have reached the disconnected-state queue
gate.SetResult();
ExpectMsg<SubscribeTagsResponse>(r => r.CorrelationId == "99");
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
Assert.Single(adapter.ReceivedCalls(), c => c.GetMethodInfo().Name == "SubscribeAsync");
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void DisconnectedWrites_AreRejectedAndNeverReplayed(bool reconnect)
{
var adapter = Adapter();
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
if (!reconnect) adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>()).Returns(gate.Task);
var actor = Actor(adapter);
if (reconnect)
{
Connected();
adapter.ConnectAsync(Arg.Any<IDictionary<string, string>>(), Arg.Any<CancellationToken>()).Returns(gate.Task);
adapter.Disconnected += Raise.Event<Action>();
AwaitCondition(() => _health.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "UpdateConnectionHealth" && c.GetArguments()[1] is ConnectionHealth.Disconnected));
}
actor.Tell(new WriteTagRequest("single", "conn", "tag", 42, DateTimeOffset.UtcNow));
ExpectMsg<WriteTagResponse>(r => !r.Success && r.CorrelationId == "single", TimeSpan.FromSeconds(1));
actor.Tell(new WriteTagBatchRequest("batch", "conn", new Dictionary<string, object?> { ["tag"] = 42 }, null, null, DateTimeOffset.UtcNow));
ExpectMsg<WriteTagBatchResponse>(r => !r.Success && r.CorrelationId == "batch", TimeSpan.FromSeconds(1));
gate.SetResult();
actor.Tell(Subscribe());
ExpectMsg<SubscribeTagsResponse>();
Assert.DoesNotContain(adapter.ReceivedCalls(), c => c.GetMethodInfo().Name is "WriteAsync" or "WriteBatchAsync");
}
[Fact]
public void SubscriberTermination_ReleasesLastTagHandle()
{
var adapter = Adapter();
var actor = Actor(adapter);
var subscriber = CreateTestProbe();
actor.Tell(Subscribe(), subscriber.Ref);
subscriber.ExpectMsg<SubscribeTagsResponse>();
Sys.Stop(subscriber.Ref);
AwaitCondition(() => adapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "UnsubscribeAsync" && Equals(c.GetArguments()[0], "handle")), TimeSpan.FromSeconds(1));
}
[Fact]
public void OldOwnerUnsubscribe_DoesNotRemoveReplacementTags()
{
var adapter = Adapter();
SubscriptionCallback? callback = null;
adapter.SubscribeAsync("tag", Arg.Do<SubscriptionCallback>(c => callback = c), Arg.Any<CancellationToken>()).Returns("handle");
var actor = Actor(adapter);
var old = CreateTestProbe();
var current = CreateTestProbe();
actor.Tell(Subscribe("old"), old.Ref);
old.ExpectMsg<SubscribeTagsResponse>();
actor.Tell(Subscribe("new"), current.Ref);
current.ExpectMsg<SubscribeTagsResponse>();
actor.Tell(new UnsubscribeTagsRequest("u", "instance", "conn", DateTimeOffset.UtcNow), old.Ref);
actor.Tell(Subscribe("barrier"), current.Ref);
current.ExpectMsg<SubscribeTagsResponse>();
callback!("tag", new TagValue(7, QualityCode.Good, DateTimeOffset.UtcNow));
current.ExpectMsg<TagValueUpdate>(u => Equals(u.Value, 7));
Assert.DoesNotContain(adapter.ReceivedCalls(), c => c.GetMethodInfo().Name == "UnsubscribeAsync");
}
[Fact]
public void DuplicateCompletion_WithLiveHandle_DoesNotUnsubscribeIt()
{
var adapter = Adapter();
var actor = Actor(adapter);
actor.Tell(Subscribe());
ExpectMsg<SubscribeTagsResponse>();
actor.Tell(new DataConnectionActor.SubscribeCompleted(Subscribe("duplicate"), TestActor,
[new DataConnectionActor.SubscribeTagResult("tag", false, true, "handle", null)], [], 0, adapter));
ExpectMsg<SubscribeTagsResponse>();
Assert.DoesNotContain(adapter.ReceivedCalls(), c => c.GetMethodInfo().Name == "UnsubscribeAsync");
}
[Fact]
public void SourceScopedSnapshotEnd_IsNotBroadcastToOtherSources()
{
var adapter = Adapter();
var alarm = (IAlarmSubscribableConnection)adapter;
AlarmTransitionCallback? callback = null;
alarm.SubscribeAlarmsAsync(Arg.Any<string>(), Arg.Any<string?>(), Arg.Do<AlarmTransitionCallback>(c => callback = c), Arg.Any<CancellationToken>()).Returns("alarm-handle");
var actor = Actor(adapter);
var a = CreateTestProbe(); var b = CreateTestProbe();
actor.Tell(new SubscribeAlarmsRequest("a", "a", "conn", "A", null, DateTimeOffset.UtcNow), a.Ref);
a.ExpectMsg<SubscribeAlarmsResponse>();
actor.Tell(new SubscribeAlarmsRequest("b", "b", "conn", "B", null, DateTimeOffset.UtcNow), b.Ref);
b.ExpectMsg<SubscribeAlarmsResponse>();
callback!(new NativeAlarmTransition("", "B", "", AlarmTransitionKind.SnapshotComplete,
new AlarmConditionState(false, false, null, AlarmShelveState.Unshelved, false, 0), "", "", "", "", "", null, DateTimeOffset.UtcNow, "", ""));
b.ExpectMsg<NativeAlarmTransitionUpdate>();
a.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
}
tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/MxGatewayBatchSeamTests.cs
+24
View File
@@ -26,6 +26,30 @@ public class MxGatewayBatchSeamTests
["ReadTimeoutMs"] = "2000",
};
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task UnsubscribeOldHandle_PreservesNewHandleRouting(bool batch)
{
var fake = new FakeMxGatewayClient();
await using var adapter = NewAdapter(fake);
await adapter.ConnectAsync(Details());
var received = new List<object?>();
var old = await adapter.SubscribeAsync("A.x", (_, value) => received.Add(value.Value));
await adapter.SubscribeAsync("A.x", (_, value) => received.Add(value.Value));
if (batch) await adapter.UnsubscribeBatchAsync([old]);
else await adapter.UnsubscribeAsync(old);
await WaitUntilAsync(() => fake.OnUpdate != null);
fake.OnUpdate!(new MxValueUpdate("A.x", 7, QualityCode.Good, DateTimeOffset.UtcNow));
Assert.Equal([7], received);
}
private static async Task WaitUntilAsync(Func<bool> condition)
{
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(2));
while (!condition()) await Task.Delay(10, timeout.Token);
}
[Fact]
public async Task SubscribeBatch_IssuesExactlyOneBulkCallPerBatch()
{
tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/Adapters/RealOpcUaSnapshotScopeTests.cs
+32
View File
@@ -0,0 +1,32 @@
using System.Reflection;
using Opc.Ua;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Alarms;
using ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Adapters;
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests.Adapters;
public class RealOpcUaSnapshotScopeTests
{
[Fact]
public void RefreshBoundaries_PreserveSourceAndHaveDistinctEpochs()
{
var client = new RealOpcUaClient();
var events = new List<NativeAlarmTransition>();
var handler = typeof(RealOpcUaClient).GetMethod("HandleAlarmEvent", BindingFlags.Instance | BindingFlags.NonPublic)!;
void Emit(string handle, string source, NodeId type) => handler.Invoke(client,
[handle, source, new EventFieldList { EventFields = new VariantCollection { new(type), new(), new(), new(), new(), new() } }, (Action<NativeAlarmTransition>)events.Add]);
Emit("a", "A", ObjectTypeIds.RefreshStartEventType);
Emit("b", "B", ObjectTypeIds.RefreshStartEventType);
Emit("a", "A", ObjectTypeIds.RefreshEndEventType);
Emit("b", "B", ObjectTypeIds.RefreshEndEventType);
Assert.Equal(4, events.Count);
Assert.Equal(["SnapshotStart", "SnapshotStart", "SnapshotComplete", "SnapshotComplete"], events.Select(e => e.Kind.ToString()));
Assert.Equal(["A", "B", "A", "B"], events.Select(e => e.SourceObjectReference));
var epoch = typeof(NativeAlarmTransition).GetProperty("SnapshotId")!;
Assert.NotNull(epoch);
Assert.NotNull(epoch.GetValue(events[0]));
Assert.Equal(epoch.GetValue(events[0]), epoch.GetValue(events[2]));
Assert.Equal(epoch.GetValue(events[1]), epoch.GetValue(events[3]));
Assert.NotEqual(epoch.GetValue(events[0]), epoch.GetValue(events[1]));
}
}
tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/DataConnectionActorTests.cs
+9 -3
View File
@@ -545,15 +545,21 @@ public class DataConnectionActorTests : TestKit
backupAdapter.ReceivedCalls().Any(c => c.GetMethodInfo().Name == "ConnectAsync"),
TimeSpan.FromSeconds(5));
// Release the in-flight subscribe; the loop resumes and subscribes t2.
// Demand is registered before I/O (DC-1), so failover independently recovers
// both tags on backup even while the original primary operation is parked.
AwaitCondition(() => backupAdapter.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "SubscribeAsync") == 2,
TimeSpan.FromSeconds(5));
// Release the in-flight subscribe; its own loop must still finish on primary.
subscribeGate.SetResult("sub-primary-1");
// t2 must go to the ORIGINAL (primary) adapter, never the swapped-in backup.
AwaitCondition(() =>
primaryAdapter.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "SubscribeAsync") >= 2,
TimeSpan.FromSeconds(5));
Assert.DoesNotContain(backupAdapter.ReceivedCalls(),
c => c.GetMethodInfo().Name == "SubscribeAsync");
Assert.Contains(primaryAdapter.ReceivedCalls(),
c => c.GetMethodInfo().Name == "SubscribeAsync" && Equals(c.GetArguments()[0], "t2"));
Assert.Equal(2, backupAdapter.ReceivedCalls().Count(c => c.GetMethodInfo().Name == "SubscribeAsync"));
}
// ── PLAN-03 Task 17 (P3): tag→instances reverse index for the value fan-out hot path ──
tests/ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests/OpcUaAlarmLiveSmokeTests.cs
+58 -2
View File
@@ -15,8 +15,9 @@ namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer.Tests;
/// <c>infra/docker-compose.yml</c>). Marked <c>[SkippableFact]</c> so it reports
/// Skipped — not failed — when the endpoint is unreachable OR does not expose
/// Alarms &amp; Conditions (no ConditionRefresh snapshot arrives). The OPC PLC
/// simulator does not reliably expose A&amp;C; see
/// <c>docs/test_infra/test_infra.md</c> for the alarm-capable-server requirement.
/// fixture exposes two Boiler event sources used by the multi-source regression.
/// That regression skips only when the endpoint is unavailable; missing fixture
/// nodes or incomplete refresh frames fail once the server is reachable.
///
/// The pure field→transition mapping is covered without a server by
/// <see cref="Adapters.OpcUaAlarmMapperTests"/>; this test proves the live
@@ -78,4 +79,59 @@ public class OpcUaAlarmLiveSmokeTests
await adapter.DisconnectAsync();
}
}
[SkippableFact]
public async Task TwoBoilerSources_KeepDistinctRefreshScopesAndEpochs()
{
await using var client = new RealOpcUaClient();
using var deadline = new CancellationTokenSource(TimeSpan.FromSeconds(20));
try
{
await client.ConnectAsync(EndpointUrl,
new OpcUaConnectionOptions(OperationTimeoutMs: 5000), deadline.Token);
}
catch (Exception ex)
{
Skip.If(true, $"OPC UA test server unavailable: {ex.Message}");
return;
}
// Distinct event notifier objects exposed by the local OPC PLC fixture.
// Adding B requests a subscription-wide ConditionRefresh, so A receives
// another refresh as well. Each monitored item must keep its own frame.
const string sourceA = "nsu=http://microsoft.com/Opc/OpcPlc/Boiler;i=5017";
const string sourceB = "nsu=http://microsoft.com/Opc/OpcPlc/Boiler;i=15070";
var transitions = new System.Collections.Concurrent.ConcurrentQueue<NativeAlarmTransition>();
var bothComplete = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
void Capture(NativeAlarmTransition transition)
{
transitions.Enqueue(transition);
if (transitions.Any(t => t.Kind == AlarmTransitionKind.SnapshotComplete && t.SourceObjectReference == sourceA)
&& transitions.Any(t => t.Kind == AlarmTransitionKind.SnapshotComplete && t.SourceObjectReference == sourceB))
bothComplete.TrySetResult();
}
await client.CreateAlarmSubscriptionAsync(sourceA, null, Capture, deadline.Token);
await client.CreateAlarmSubscriptionAsync(sourceB, null, Capture, deadline.Token);
await bothComplete.Task.WaitAsync(deadline.Token);
var boundaries = transitions.Where(t => t.Kind is AlarmTransitionKind.SnapshotStart
or AlarmTransitionKind.SnapshotComplete).ToArray();
foreach (var source in new[] { sourceA, sourceB })
{
var frames = boundaries.Where(t => t.SourceObjectReference == source).ToArray();
Assert.Contains(frames, t => t.Kind == AlarmTransitionKind.SnapshotStart);
Assert.Contains(frames, t => t.Kind == AlarmTransitionKind.SnapshotComplete);
Assert.All(frames, t =>
{
Assert.Equal(source, t.SnapshotSourceReference);
Assert.NotNull(t.SnapshotId);
});
foreach (var end in frames.Where(t => t.Kind == AlarmTransitionKind.SnapshotComplete))
Assert.Single(frames, t => t.Kind == AlarmTransitionKind.SnapshotStart && t.SnapshotId == end.SnapshotId);
}
Assert.Empty(boundaries.Where(t => t.SourceObjectReference == sourceA).Select(t => t.SnapshotId)
.Intersect(boundaries.Where(t => t.SourceObjectReference == sourceB).Select(t => t.SnapshotId)));
}
}