Joseph Doherty
9fb52153fd
Deferred flake-pattern sweep of tests/ for the class fixed inc4caebe9andcfa6acbf— a bounded wait on observable A followed by a bare assert on an observable B that the product only reaches strictly after A. Three clear instances, each reproduced deterministically by delaying only the later step and each re-verified green with that same delay still injected. AlarmOnTriggerRuns_ShedAtTheSameCap_WithAnAlarmScopedSiteEvent gated on the rate-limited shed site event and then asserted the shed COUNT bare. AlarmActor.ShedAlarmRun increments the counter and only then emits the event, and the event fires on the first shed only — so the gate observed Flap(4)'s shed and ordered nothing with respect to Flap(5)'s, which is a separate mailbox message with no observable of its own (an alarm on-trigger run has no Ask caller to reply to, unlike ScriptActor.ShedRun, whose sibling test is correctly ordered by its ScriptCallResult and is left alone). Deferring Flap(5) by 2 s failed it with "Expected: 2 / Actual: 1". The count is now ACCUMULATED across polls rather than re-read, because SiteHealthCollector.CollectReport DRAINS the interval counters — a poll loop that simply re-read it would consume the first shed and never reach 2. EndToEnd_GrpcStubError_RowStays_Pending_NextTick_Succeeds gated on the central row arriving and then asserted bare that the site SQLite row had left Pending. SiteAuditTelemetryActor pushes via IngestAuditEventsAsync (which is what writes the central row) and calls MarkForwardedAsync only after parsing the ack. Delaying just that post-push step failed it with "Assert.DoesNotContain() Failure: Filter matched in collection". PreSnapshotBuffer_IsCapped_DropsOldest_AndCountsTheDrops gated on "Count >= cap" and then asserted "Count == cap + 1" bare — a gate strictly weaker than the assertion it guards, so it ordered nothing with respect to the last event of a FlushBuffer loop that delivers one at a time. Parking that loop after its 19,999th delivery failed it with "Expected: 20001 / Actual: 20000". Also hardens GrpcCentralTransportTests.WaitUntil, which returned silently on timeout; today's single caller re-asserts immediately, so this only sharpens the message rather than fixing a live flake. Cleared with evidence, not guessed: SiteAlarmLiveCacheService's LingerStop removes the site entry inside one lock, so IsLive and GetCurrentAlarms flip atomically; and SiteReconciliationActor walks response.Gap with a sequential foreach in which the asserted "Gone" log precedes the awaited "Good" row, the inverse of this class. Test-only; every ordering named above is correct as written.
1320 lines
57 KiB
C#
1320 lines
57 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Akka.TestKit.Xunit2;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
|
|
|
/// <summary>
|
|
/// Tests for DebugStreamBridgeActor with gRPC streaming integration.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamServiceTests</c>:
|
|
/// both tune the actor's static test seams (<c>SnapshotTimeout</c>, <c>ConsumerIdleTimeout</c>,
|
|
/// …), and xUnit parallelizes distinct classes by default, which would let one class's
|
|
/// try/finally restore clobber the other's window mid-test.
|
|
/// </remarks>
|
|
[Collection("DebugStreamStatics")]
|
|
public class DebugStreamBridgeActorTests : TestKit
|
|
{
|
|
private const string SiteId = "site-alpha";
|
|
private const string InstanceName = "Site1.Pump01";
|
|
private const string GrpcNodeA = "http://localhost:5100";
|
|
private const string GrpcNodeB = "http://localhost:5200";
|
|
|
|
public DebugStreamBridgeActorTests() : base(@"akka.loglevel = DEBUG")
|
|
{
|
|
// Use a very short reconnect delay for testing
|
|
DebugStreamBridgeActor.ReconnectDelay = TimeSpan.FromMilliseconds(100);
|
|
// Long stability window so streams are never considered "stable" mid-test
|
|
// unless a test deliberately waits it out.
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
|
}
|
|
|
|
private record TestContext(
|
|
IActorRef BridgeActor,
|
|
TestProbe CommProbe,
|
|
MockSiteStreamGrpcClient MockGrpcClient,
|
|
List<object> ReceivedEvents,
|
|
bool[] TerminatedFlag);
|
|
|
|
private TestContext CreateBridgeActor()
|
|
{
|
|
var commProbe = CreateTestProbe();
|
|
var mockClient = new MockSiteStreamGrpcClient();
|
|
var factory = new MockSiteStreamGrpcClientFactory(mockClient);
|
|
var events = new List<object>();
|
|
var terminated = new[] { false };
|
|
|
|
Action<object> onEvent = evt => { lock (events) { events.Add(evt); } };
|
|
Action onTerminated = () => terminated[0] = true;
|
|
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId,
|
|
InstanceName,
|
|
"corr-1",
|
|
commProbe.Ref,
|
|
onEvent,
|
|
onTerminated,
|
|
factory,
|
|
GrpcNodeA,
|
|
GrpcNodeB);
|
|
|
|
var actor = Sys.ActorOf(props);
|
|
return new TestContext(actor, commProbe, mockClient, events, terminated);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_InstanceNotFound_Snapshot_Forwards_To_OnEvent_Tears_Down_Stream_And_Terminates()
|
|
{
|
|
// M2.11 (revised for M2.18 stream-first): the gRPC subscription is now opened
|
|
// up-front in PreStart, so when the site reports InstanceNotFound=true the
|
|
// bridge actor must
|
|
// (a) forward the not-found snapshot to _onEvent so DebugStreamService's TCS
|
|
// resolves and the caller can inspect the flag,
|
|
// (b) tear DOWN the already-opened gRPC stream (Unsubscribe the just-opened
|
|
// correlation) rather than enter pass-through, and
|
|
// (c) stop itself cleanly.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // initial subscribe envelope
|
|
|
|
// Stream-first: the gRPC subscription is opened before the snapshot arrives.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var notFoundSnapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow,
|
|
InstanceNotFound: true);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(notFoundSnapshot);
|
|
|
|
// (a) _onEvent must receive the not-found snapshot
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
var received = Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.True(received.InstanceNotFound);
|
|
}
|
|
|
|
// (b) the just-opened gRPC stream is torn down (not left running / no pass-through)
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// (c) actor terminates cleanly
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
[Fact]
|
|
public void PreStart_Sends_SubscribeDebugViewRequest_Via_ClusterClient()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.Equal(SiteId, envelope.SiteId);
|
|
Assert.IsType<SubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
var req = (SubscribeDebugViewRequest)envelope.Message;
|
|
Assert.Equal(InstanceName, req.InstanceUniqueName);
|
|
Assert.Equal("corr-1", req.CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Forwards_To_OnEvent_Callback()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Does_Not_Open_Additional_GrpcStream()
|
|
{
|
|
// M2.18 stream-first: the gRPC subscription is opened in PreStart, BEFORE the
|
|
// snapshot arrives. After the snapshot is delivered the actor switches to
|
|
// pass-through — it must NOT open a second subscription. Exactly ONE subscribe
|
|
// call should have been made (the PreStart one).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
// Verify the stream is already open before the snapshot.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// After snapshot delivery, still exactly ONE subscribe — no additional stream opened.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
var singleCall = Assert.Single(ctx.MockGrpcClient.SubscribeCalls);
|
|
Assert.Equal("corr-1", singleCall.CorrelationId);
|
|
Assert.Equal(InstanceName, singleCall.InstanceUniqueName);
|
|
}
|
|
|
|
[Fact]
|
|
public void Events_From_GrpcCallback_Forwarded_To_OnEvent()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC event arriving via the onEvent callback
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(attrChange);
|
|
|
|
// snapshot + attr change
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Reconnects_To_Other_Node()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC error
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Stream broken"));
|
|
|
|
// Should resubscribe
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("corr-1", ctx.MockGrpcClient.SubscribeCalls[1].CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Unsubscribes_Old_Stream_Before_Reconnect()
|
|
{
|
|
// Communication-002 regression: a reconnect must unsubscribe the previous
|
|
// stream so the old node does not keep a zombie relay actor / subscription.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC error → reconnect
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Stream broken"));
|
|
|
|
// Should resubscribe...
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// ...and must have unsubscribed the prior correlation ID so the old node's
|
|
// relay actor is released rather than left zombie.
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
}
|
|
|
|
[Fact]
|
|
public void After_MaxRetries_Terminates()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// 4 consecutive errors: initial + 3 retries, then terminate
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Error 1"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnError(new Exception("Error 2"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 3, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[2].OnError(new Exception("Error 3"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 4, TimeSpan.FromSeconds(5));
|
|
|
|
// Fourth error exceeds max retries
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Error 4"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void StopDebugStream_Cancels_Grpc_And_Sends_Unsubscribe()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
ctx.BridgeActor.Tell(new StopDebugStream());
|
|
|
|
// Should send the site-addressed unsubscribe
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.IsType<UnsubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
// Should unsubscribe gRPC
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Count > 0, TimeSpan.FromSeconds(3));
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
|
|
// Should stop self
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
}
|
|
|
|
[Fact]
|
|
public void DebugStreamTerminated_Stops_Actor_Idempotently()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(new DebugStreamTerminated(SiteId, "corr-1"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void FlappingStream_DeliveringEventsBetweenFailures_StillTerminatesAfterMaxRetries()
|
|
{
|
|
// Communication-008 regression: a stream that connects, delivers an event,
|
|
// then fails — repeatedly — must still trip MaxRetries. The retry count is
|
|
// NO LONGER reset by a received event (only by the stability window). The
|
|
// previous behaviour reset _retryCount on every event, so a flapping site
|
|
// reconnected forever and the debug session lived on indefinitely.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
|
|
// Flap: deliver one event then fail, three times. Each event would, under
|
|
// the old buggy logic, reset the retry budget and prevent termination.
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
var call = ctx.MockGrpcClient.SubscribeCalls[i];
|
|
call.OnEvent(attrChange);
|
|
call.OnError(new Exception($"Flap {i + 1}"));
|
|
var expected = i + 2;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
// Fourth error (after the 3 retries) must exceed MaxRetries and terminate.
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnEvent(attrChange);
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Flap 4"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Reconnects_To_Other_Node_Endpoint()
|
|
{
|
|
// Communication-015 regression: drive the bridge actor through a node flip
|
|
// with an endpoint-aware factory (one distinct mock client per endpoint).
|
|
// The first subscribe targets NodeA; after a gRPC error the bridge must
|
|
// reconnect via a client bound to the *NodeB* endpoint.
|
|
var commProbe = CreateTestProbe();
|
|
var factory = new EndpointTrackingGrpcClientFactory();
|
|
var events = new List<object>();
|
|
var terminated = new[] { false };
|
|
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId, InstanceName, "corr-1", commProbe.Ref,
|
|
(Action<object>)(evt => { lock (events) { events.Add(evt); } }),
|
|
(Action)(() => terminated[0] = true),
|
|
factory, GrpcNodeA, GrpcNodeB);
|
|
|
|
var actor = Sys.ActorOf(props);
|
|
commProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
actor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// First subscribe goes to NodeA.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// gRPC error → bridge flips to NodeB.
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnError(new Exception("NodeA down"));
|
|
|
|
// The reconnect must reach a client bound to the NodeB endpoint.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
Assert.Equal("corr-1", factory.ClientFor(GrpcNodeB).SubscribeCalls[0].CorrelationId);
|
|
}
|
|
|
|
// ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
|
|
|
|
[Fact]
|
|
public void On_GracefulStreamCompletion_Reopens_On_The_Same_Node()
|
|
{
|
|
// The site caps every stream at GrpcMaxStreamLifetime and then ends the RPC with
|
|
// OK. Pre-fix the client's read loop just finished, the actor was told nothing, and
|
|
// the debug session went silently deaf for the rest of its life.
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnCompleted();
|
|
|
|
// Reopened on the SAME node — a clean close is not a fault, so there is nothing to
|
|
// fail over from — and the finished stream is released, not left zombie.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 2,
|
|
TimeSpan.FromSeconds(5));
|
|
Assert.Empty(factory.ClientFor(GrpcNodeB).SubscribeCalls);
|
|
Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds);
|
|
}
|
|
|
|
[Fact]
|
|
public void RepeatedGracefulCompletions_DoNotConsume_TheErrorRetryBudget()
|
|
{
|
|
// Five completions — two more than MaxRetries. Had completion been routed through
|
|
// the error path the session would have terminated on the fourth.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
for (var i = 1; i <= 5; i++)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[i - 1].OnCompleted();
|
|
var expected = i + 1;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected,
|
|
TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void LateCompletionFromAPreviousStreamGeneration_IsIgnored()
|
|
{
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
var firstSub = factory.ClientFor(GrpcNodeA).SubscribeCalls[0];
|
|
firstSub.OnError(new Exception("NodeA down")); // gen 1 dies → gen 2 opens on NodeB
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
|
|
// A completion racing out of the dead gen-1 stream must not tear down the live one.
|
|
firstSub.OnCompleted();
|
|
Thread.Sleep(300);
|
|
Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).UnsubscribedCorrelationIds);
|
|
Assert.Single(factory.ClientFor(GrpcNodeB).SubscribeCalls);
|
|
}
|
|
|
|
// ── Task 6 (arch review 02, High): teardown/failover unsubscribe is endpoint-safe ──
|
|
// Both paths use TryGet, never GetOrCreate, so cleanup can never open a fresh
|
|
// channel or (with (site,endpoint) keying) touch another session's channel.
|
|
|
|
private (IActorRef Actor, EndpointTrackingGrpcClientFactory Factory) CreateBridgeWithTrackingFactory()
|
|
{
|
|
var commProbe = CreateTestProbe();
|
|
var factory = new EndpointTrackingGrpcClientFactory();
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId, InstanceName, "corr-1", commProbe.Ref,
|
|
(Action<object>)(_ => { }), (Action)(() => { }),
|
|
factory, GrpcNodeA, GrpcNodeB);
|
|
var actor = Sys.ActorOf(props);
|
|
commProbe.ExpectMsg<SiteEnvelope>(); // PreStart subscribe handshake
|
|
return (actor, factory);
|
|
}
|
|
|
|
[Fact]
|
|
public void CleanupGrpc_UnsubscribesViaTryGet_WithoutOpeningAChannel()
|
|
{
|
|
var (actor, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
var createdBefore = factory.CreatedCount; // node-a only
|
|
|
|
actor.Tell(new StopDebugStream());
|
|
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
// Pre-fix CleanupGrpc called GetOrCreate → could open a fresh channel here.
|
|
Assert.Equal(createdBefore, factory.CreatedCount);
|
|
}
|
|
|
|
[Fact]
|
|
public void SessionFailover_UnsubscribesFailedEndpointViaTryGet_OpensOnlyTheReconnectChannel()
|
|
{
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Fail NodeA: the bridge unsubscribes the failed NodeA stream via TryGet
|
|
// (if that path had used GetOrCreate on the post-flip cache it could have
|
|
// disposed/created the wrong client), then reconnects to NodeB.
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnError(new Exception("NodeA down"));
|
|
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(5));
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
// Only the two real node endpoints were ever opened — the unsubscribe
|
|
// itself created nothing.
|
|
Assert.Equal(2, factory.CreatedCount);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// M2.18 (#26) — stream-first + replay/dedup
|
|
// ---------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void PreStart_Opens_GrpcStream_Before_Snapshot_Arrives()
|
|
{
|
|
// M2.18: the gRPC subscription must be opened in PreStart (stream-first),
|
|
// BEFORE the snapshot is delivered, so live events start flowing during the
|
|
// snapshot-build + network-transit window. The old lifecycle opened the
|
|
// stream only after the snapshot arrived, losing gap-window events.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // initial subscribe envelope
|
|
|
|
// No snapshot sent yet — the stream must already be open.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Assert.Equal("corr-1", ctx.MockGrpcClient.SubscribeCalls[0].CorrelationId);
|
|
Assert.Equal(InstanceName, ctx.MockGrpcClient.SubscribeCalls[0].InstanceUniqueName);
|
|
|
|
// _onEvent must NOT have fired — buffering, not delivering.
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
}
|
|
|
|
[Fact]
|
|
public void GapWindow_Event_Buffered_Before_Snapshot_Is_Delivered_Exactly_Once_After_Snapshot()
|
|
{
|
|
// M2.18: an event arriving DURING the snapshot window (before the snapshot
|
|
// is delivered) is buffered, then flushed exactly once AFTER the snapshot.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Live event arrives BEFORE the snapshot — its entity is NOT in the snapshot,
|
|
// so it is a genuine gap-window event that must survive.
|
|
var gapEvent = new AttributeValueChanged(InstanceName, "IO", "Pressure", 99.9, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(gapEvent);
|
|
|
|
// While buffering, _onEvent has not fired.
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot then the buffered gap-window event, exactly once, in that order.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal("Pressure", flushed.AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Event_Already_Reflected_In_Snapshot_Is_Dropped()
|
|
{
|
|
// M2.18 dedup: a buffered event whose entity is in the snapshot with an equal
|
|
// or newer snapshot timestamp (buffered.Timestamp <= snapshot.Timestamp) is
|
|
// already reflected and must be DROPPED.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var t0 = DateTimeOffset.UtcNow;
|
|
|
|
// Buffered event for "Temp" at t0.
|
|
var buffered = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", t0);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(buffered);
|
|
|
|
// Snapshot already contains "Temp" at the SAME timestamp t0 → buffered is a dup.
|
|
var snapAttr = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", t0);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged> { snapAttr },
|
|
new List<AlarmStateChanged>(),
|
|
t0);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// Only the snapshot is delivered; the buffered duplicate is dropped.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
// Give a beat to ensure no extra (dropped) event sneaks through.
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Single(ctx.ReceivedEvents);
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Event_Strictly_Newer_Than_Snapshot_Entity_Is_Delivered()
|
|
{
|
|
// M2.18 dedup: a buffered event strictly newer than the snapshot's entry for
|
|
// the same entity (buffered.Timestamp > snapshot.Timestamp) is NOT a dup and
|
|
// must be DELIVERED after the snapshot.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapTime = DateTimeOffset.UtcNow;
|
|
var newerTime = snapTime.AddMilliseconds(1);
|
|
|
|
// Buffered event for "Temp" strictly NEWER than the snapshot's "Temp".
|
|
var buffered = new AttributeValueChanged(InstanceName, "IO", "Temp", 50.0, "Good", newerTime);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(buffered);
|
|
|
|
var snapAttr = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", snapTime);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged> { snapAttr },
|
|
new List<AlarmStateChanged>(),
|
|
snapTime);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot then the strictly-newer buffered event.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal(50.0, flushed.Value);
|
|
Assert.Equal(newerTime, flushed.Timestamp);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Alarm_Dedup_Uses_AlarmIdentity_And_Timestamp()
|
|
{
|
|
// M2.18 dedup for alarms: identity = (instance, alarm name, source reference).
|
|
// A buffered alarm older-or-equal to the snapshot's same-identity alarm is
|
|
// dropped; a strictly-newer one is delivered.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var t0 = DateTimeOffset.UtcNow;
|
|
|
|
// Buffered: "PumpFault" at t0 (dup) and "Overheat" at t0+1ms (newer, deliver).
|
|
var dupAlarm = new AlarmStateChanged(InstanceName, "PumpFault",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 500, t0);
|
|
var newerAlarm = new AlarmStateChanged(InstanceName, "Overheat",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 700, t0.AddMilliseconds(1));
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(dupAlarm);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(newerAlarm);
|
|
|
|
// Snapshot contains BOTH "PumpFault" and "Overheat" at t0.
|
|
var snapPumpFault = new AlarmStateChanged(InstanceName, "PumpFault",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 500, t0);
|
|
var snapOverheat = new AlarmStateChanged(InstanceName, "Overheat",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Normal, 0, t0);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged> { snapPumpFault, snapOverheat },
|
|
t0);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot + only the strictly-newer "Overheat" alarm (PumpFault dropped).
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Equal(2, ctx.ReceivedEvents.Count);
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AlarmStateChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal("Overheat", flushed.AlarmName);
|
|
Assert.Equal(700, flushed.Priority);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Events_Flushed_In_Arrival_Order()
|
|
{
|
|
// M2.18: ordering preserved across multiple buffered events (none are dups —
|
|
// their entities are absent from the snapshot).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var baseTime = DateTimeOffset.UtcNow;
|
|
var sub = ctx.MockGrpcClient.SubscribeCalls[0];
|
|
sub.OnEvent(new AttributeValueChanged(InstanceName, "IO", "A", 1, "Good", baseTime));
|
|
sub.OnEvent(new AlarmStateChanged(InstanceName, "AlarmX",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 100, baseTime));
|
|
sub.OnEvent(new AttributeValueChanged(InstanceName, "IO", "B", 2, "Good", baseTime));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
baseTime);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 4; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.Equal("A", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
Assert.Equal("AlarmX", Assert.IsType<AlarmStateChanged>(ctx.ReceivedEvents[2]).AlarmName);
|
|
Assert.Equal("B", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[3]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void PassThrough_After_Flush_Delivers_Subsequent_Events_Immediately()
|
|
{
|
|
// M2.18: after the snapshot+flush the actor switches to pass-through — later
|
|
// events go straight to _onEvent (no buffering, no dup).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Post-snapshot event — must be delivered immediately, exactly once.
|
|
var postEvent = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(postEvent);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void InstanceNotFound_After_StreamFirst_Tears_Down_Stream_And_Does_Not_PassThrough()
|
|
{
|
|
// M2.18 + M2.11: stream-first means the gRPC subscription is already open
|
|
// when an InstanceNotFound snapshot arrives. The bridge must tear that stream
|
|
// down (Unsubscribe the just-opened correlation), deliver the not-found
|
|
// snapshot, NOT enter pass-through, and stop cleanly.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
// Stream opened up-front (stream-first).
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var notFoundSnapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow,
|
|
InstanceNotFound: true);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(notFoundSnapshot);
|
|
|
|
// Not-found snapshot delivered.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.True(Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]).InstanceNotFound);
|
|
}
|
|
|
|
// The just-opened stream must be torn down.
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Stops cleanly.
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
|
|
// No pass-through: an event arriving after the stop is not delivered.
|
|
var late = new AttributeValueChanged(InstanceName, "IO", "Temp", 1, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(late);
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents) { Assert.Single(ctx.ReceivedEvents); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Reconnect_During_Buffering_Phase_Keeps_Buffering_Until_Snapshot()
|
|
{
|
|
// M2.18: a gRPC error/reconnect BEFORE the snapshot arrives must remain in the
|
|
// buffering phase — events on the new stream are still buffered, then flushed
|
|
// when the snapshot finally arrives.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Error before snapshot → reconnect (still buffering).
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("pre-snapshot blip"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Event on the reconnected stream — still buffered (snapshot not yet delivered).
|
|
var gapEvent = new AttributeValueChanged(InstanceName, "IO", "Late", 7, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnEvent(gapEvent);
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot + the event buffered across the reconnect.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.Equal("Late", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Reconnect_After_Snapshot_Resumes_PassThrough_Not_Buffering()
|
|
{
|
|
// M2.18: a mid-session reconnect (after the snapshot was already delivered)
|
|
// must resume pass-through — the snapshot is a one-time thing and events on
|
|
// the reconnected stream are delivered immediately, not re-buffered.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Mid-session reconnect.
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("mid-session blip"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Event on the reconnected stream — delivered immediately (pass-through).
|
|
var postEvent = new AttributeValueChanged(InstanceName, "IO", "Temp", 9, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnEvent(postEvent);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Equal("Temp", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void RetryCount_RecoveredOnlyAfterStreamStaysStableForStabilityWindow()
|
|
{
|
|
// Communication-008: after a stream has been connected for the stability
|
|
// window, the retry budget is recovered — a later transient fault then gets
|
|
// a fresh set of retries rather than being counted against the old budget.
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Two failures — but each new stream stays up long enough (the mock
|
|
// stream only completes on cancel) for the stability window to elapse
|
|
// and reset the retry budget before the next failure.
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
Thread.Sleep(450); // exceed the 300ms stability window
|
|
ctx.MockGrpcClient.SubscribeCalls[i].OnError(new Exception($"Error {i + 1}"));
|
|
var expected = i + 2;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
// Five well-spaced failures did NOT terminate the actor because each
|
|
// reconnect recovered its retry budget after the stability window.
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
// ── WP2.3: bounded pre-snapshot buffer, hard snapshot deadline, timeout hygiene ──
|
|
|
|
[Fact]
|
|
public void PreSnapshotBuffer_IsCapped_DropsOldest_AndCountsTheDrops()
|
|
{
|
|
// Before the cap a snapshot that never arrived buffered every live event on the
|
|
// CENTRAL node without limit — one wedged session on a chatty instance was enough
|
|
// to grow unbounded. Now the oldest are evicted and counted.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe envelope; no snapshot is ever sent
|
|
|
|
const int cap = 20_000;
|
|
const int overflow = 250;
|
|
var before = Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped);
|
|
|
|
var t = DateTimeOffset.UtcNow;
|
|
for (var i = 0; i < cap + overflow; i++)
|
|
{
|
|
ctx.BridgeActor.Tell(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", $"Attr{i}", i, "Good", t.AddMilliseconds(i)));
|
|
}
|
|
|
|
// The overflow was evicted (drop-oldest) and counted.
|
|
AwaitCondition(
|
|
() => Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped) - before >= overflow,
|
|
TimeSpan.FromSeconds(10));
|
|
|
|
// The session is still healthy: the snapshot can still arrive and flush the
|
|
// (capped) buffer — the newest events, which the snapshot may predate, survived.
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
t.AddMilliseconds(-1));
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// One awaited block, gated on the EXACT final count. This used to gate on
|
|
// "Count >= cap" and then assert "Count == cap + 1" bare — a gate strictly
|
|
// weaker than the assertion it guards, which therefore orders nothing with
|
|
// respect to the last event. FlushBuffer delivers the buffered events one
|
|
// by one via _onEvent inside a single loop, so the poll can legitimately
|
|
// observe the count crossing `cap` while the loop still has an event to
|
|
// go. Reproduced deterministically by parking that loop for 3 s after its
|
|
// 19,999th delivery: the bare form failed with "Expected: 20001 / Actual:
|
|
// 20000". The claims are unchanged — exactly the snapshot plus the capped
|
|
// retained events, with the newest survivor last.
|
|
AwaitAssert(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
// Snapshot + exactly the retained (capped) events, and the newest survived.
|
|
Assert.Equal(cap + 1, ctx.ReceivedEvents.Count);
|
|
var lastAttr = ctx.ReceivedEvents.OfType<AttributeValueChanged>().Last();
|
|
Assert.Equal($"Attr{cap + overflow - 1}", lastAttr.AttributeName);
|
|
}
|
|
}, TimeSpan.FromSeconds(10));
|
|
}
|
|
|
|
[Fact]
|
|
public void NoSnapshotWithinDeadline_FailsTheSession_InsteadOfBufferingForever()
|
|
{
|
|
// Nothing else ends a session wedged in the buffering phase: a lost site reply
|
|
// raises no gRPC error, and stream events no longer reset the orphan timeout.
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe request — never answered
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
|
|
// The consumer is told, so the UI can surface the failure and reopen.
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
// And the site-side relay was released rather than left as a zombie.
|
|
AwaitCondition(
|
|
() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void SnapshotArrival_StandsDownTheDeadline()
|
|
{
|
|
// The deadline must not fire after a healthy snapshot — a live session would
|
|
// otherwise be killed mid-stream.
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
Thread.Sleep(700); // well past the deadline
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// Still serving: a post-snapshot event passes straight through.
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AlarmStateChanged(
|
|
InstanceName, "PumpFault", Commons.Types.Enums.AlarmState.Active, 500,
|
|
DateTimeOffset.UtcNow));
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AlarmStateChanged>().Any(); }
|
|
}, TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void StreamEvents_StillReachTheConsumer_ThroughTheWrappedCallbackPath()
|
|
{
|
|
// The gRPC callback wraps every event in LiveDebugStreamEvent; the wrapper must be
|
|
// transparent to delivery (it exists only to keep the high-volume path explicit —
|
|
// the orphan net keys off the consumer keepalive, not the mailbox).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
var evt = new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
|
|
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
|
|
}, TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
// ----- Orphan net: measures the CONSUMER, not the mailbox ----- //
|
|
|
|
[Fact]
|
|
public void HealthySession_StreamingEvents_WithConsumerKeepalives_SurvivesWellPastTheOrphanWindow()
|
|
{
|
|
// THE regression this closes: with the orphan net armed off the mailbox
|
|
// (SetReceiveTimeout) and stream events correctly excluded from it, nothing recurring
|
|
// reset it — the snapshot lands once, GrpcStreamStable once — so a perfectly healthy
|
|
// session self-terminated one window later and the consumer was told
|
|
// "Site disconnected". Here the session streams events and receives the keepalive
|
|
// DebugStreamService sends while it is attached; it must live through MANY windows.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Watch(ctx.BridgeActor);
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// ~5 orphan windows of pure stream traffic + consumer keepalives, and no other
|
|
// mailbox activity whatsoever (no reconnects, no snapshots, no stop).
|
|
var deadline = DateTime.UtcNow.AddSeconds(2);
|
|
var delivered = 0;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 20.0 + delivered, "Good",
|
|
DateTimeOffset.UtcNow));
|
|
delivered++;
|
|
// What DebugStreamService's shared timer does for an attached session.
|
|
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive());
|
|
Thread.Sleep(100);
|
|
}
|
|
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// Still serving: the events all arrived and the actor is alive.
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Count() == delivered; }
|
|
}, TimeSpan.FromSeconds(3));
|
|
Assert.False(ctx.BridgeActor.IsNobody());
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void OrphanedSession_ConsumerGone_StillTerminates_EvenWhileEventsKeepArriving()
|
|
{
|
|
// The other half of the contract: site chatter must NOT hold an abandoned session
|
|
// open. No keepalive arrives (the consumer is gone), so the session terminates,
|
|
// unsubscribes from the site and reports termination — while events stream in.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Watch(ctx.BridgeActor);
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// Keep the site chatty for longer than the orphan window — with no keepalive.
|
|
var deadline = DateTime.UtcNow.AddSeconds(1);
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow));
|
|
Thread.Sleep(50);
|
|
}
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ConsumerKeepalive_RenewsTheWindow_AfterANearMiss()
|
|
{
|
|
// A single late keepalive is enough to save a session — the stamp is refreshed, not
|
|
// a one-shot grace period.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(500);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Watch(ctx.BridgeActor);
|
|
|
|
Thread.Sleep(350); // most of the window burnt
|
|
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive()); // …then the consumer checks in
|
|
Thread.Sleep(350); // past the ORIGINAL deadline
|
|
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// …and once the keepalives stop, it does terminate.
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Mock gRPC client that records SubscribeAsync and Unsubscribe calls.
|
|
/// <para>
|
|
/// <b>Thread safety:</b> <see cref="SubscribeCalls"/> and
|
|
/// <see cref="UnsubscribedCorrelationIds"/> are written from the actor/background thread
|
|
/// (via <see cref="SubscribeAsync"/> and <see cref="Unsubscribe"/>) and read from the test
|
|
/// thread (via <c>AwaitCondition</c> / assertions). All access goes through a shared lock
|
|
/// to match the <c>lock (events)</c> pattern used for <c>ctx.ReceivedEvents</c>.
|
|
/// </para>
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClient : SiteStreamGrpcClient
|
|
{
|
|
private readonly object _lock = new();
|
|
private readonly List<MockSubscription> _subscribeCalls = new();
|
|
private readonly List<string> _unsubscribedCorrelationIds = new();
|
|
|
|
/// <summary>Returns a snapshot of subscribe calls, taken under the internal lock.</summary>
|
|
public List<MockSubscription> SubscribeCalls { get { lock (_lock) { return _subscribeCalls.ToList(); } } }
|
|
|
|
/// <summary>Returns a snapshot of unsubscribed correlation IDs, taken under the internal lock.</summary>
|
|
public List<string> UnsubscribedCorrelationIds { get { lock (_lock) { return _unsubscribedCorrelationIds.ToList(); } } }
|
|
|
|
private MockSiteStreamGrpcClient(bool _) : base() { }
|
|
|
|
public MockSiteStreamGrpcClient() : base()
|
|
{
|
|
}
|
|
|
|
public override Task SubscribeAsync(
|
|
string correlationId,
|
|
string instanceUniqueName,
|
|
Action<object> onEvent,
|
|
Action<Exception> onError,
|
|
Action onCompleted,
|
|
CancellationToken ct)
|
|
{
|
|
var subscription = new MockSubscription(
|
|
correlationId, instanceUniqueName, onEvent, onError, onCompleted, ct);
|
|
lock (_lock) { _subscribeCalls.Add(subscription); }
|
|
|
|
// Return a task that completes when cancelled (simulates long-running stream)
|
|
var tcs = new TaskCompletionSource();
|
|
ct.Register(() => tcs.TrySetResult());
|
|
return tcs.Task;
|
|
}
|
|
|
|
public override void Unsubscribe(string correlationId)
|
|
{
|
|
lock (_lock) { _unsubscribedCorrelationIds.Add(correlationId); }
|
|
}
|
|
}
|
|
|
|
internal record MockSubscription(
|
|
string CorrelationId,
|
|
string InstanceUniqueName,
|
|
Action<object> OnEvent,
|
|
Action<Exception> OnError,
|
|
Action OnCompleted,
|
|
CancellationToken CancellationToken);
|
|
|
|
/// <summary>
|
|
/// Factory that always returns the pre-configured mock client.
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClientFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly MockSiteStreamGrpcClient _mockClient;
|
|
public List<string> RequestedEndpoints { get; } = new();
|
|
|
|
public MockSiteStreamGrpcClientFactory(MockSiteStreamGrpcClient mockClient)
|
|
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance)
|
|
{
|
|
_mockClient = mockClient;
|
|
}
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
{
|
|
RequestedEndpoints.Add(grpcEndpoint);
|
|
return _mockClient;
|
|
}
|
|
|
|
// Mirrors the real factory's TryGet: returns the client only for an endpoint
|
|
// already opened via GetOrCreate, never creating one. Lets teardown/failover
|
|
// unsubscribe paths (now TryGet-based, Task 6) resolve the mock client.
|
|
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
|
=> RequestedEndpoints.Contains(grpcEndpoint) ? _mockClient : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Endpoint-aware mock factory: hands out a distinct <see cref="MockSiteStreamGrpcClient"/>
|
|
/// per endpoint, mirroring the real factory's corrected NodeA→NodeB failover behaviour
|
|
/// so node-flip coverage is meaningful (Communication-015).
|
|
/// </summary>
|
|
internal class EndpointTrackingGrpcClientFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, MockSiteStreamGrpcClient> _byEndpoint = new();
|
|
|
|
public EndpointTrackingGrpcClientFactory()
|
|
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance)
|
|
{
|
|
}
|
|
|
|
public MockSiteStreamGrpcClient ClientFor(string endpoint) =>
|
|
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteStreamGrpcClient());
|
|
|
|
/// <summary>Number of distinct endpoint clients created so far (channels opened).</summary>
|
|
public int CreatedCount => _byEndpoint.Count;
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
=> ClientFor(grpcEndpoint);
|
|
|
|
// TryGet never creates: returns the client only for an already-opened endpoint.
|
|
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
|
=> _byEndpoint.TryGetValue(grpcEndpoint, out var client) ? client : null;
|
|
}
|