Joseph Doherty
fd5e023d08
F1 (HIGH) DebugStreamBridgeActor: the 5-minute orphan net measured the MAILBOX (SetReceiveTimeout), and once stream events were correctly marked INotInfluenceReceiveTimeout nothing recurring reset it — the snapshot lands once and GrpcStreamStable once — so every healthy session self-terminated at ~6 min with a false "Site disconnected". Replaced with a periodic self-tick (ConsumerLivenessCheckInterval, 30s) over a consumer-last-seen stamp renewed only by DebugStreamConsumerAlive, which DebugStreamService Tells on a shared timer to every session still in its registry (holding a session there IS "a consumer is attached" — both the Blazor view and the SignalR hub release it on dispose/disconnect, and it works headless). Reverting the wrapper was rejected: it would restore the quiet-instance orphan bug. F2 (MED) SiteStreamGrpcClient: the RpcException(Cancelled) filter now requires cts.IsCancellationRequested. A peer-originated / channel-dispose Cancelled fired none of onError/onCompleted/onConnected, leaving SiteAlarmAggregatorActor with _streamDown=false forever (IsLive stuck true, reconcile reopen guard never fired). F3 (MED) SiteStreamGrpcClient: a header TIMEOUT is no longer reported as connected — that shape is exactly what an unreachable site produces, and it cleared _streamDown, consumed _seedOnConnect and launched a full snapshot fan-out at a dead site. AwaitHeadersAsync returns bool; the first received event is the fallback connected signal, fired at most once from headers OR first event. F4 (LOW-MED) SqliteAuditWriter.MarkReconciledUpToAsync: the blanket below-cursor UPDATE retired late-stamped inserts that were never served (then age-purged — silent loss). The flip is now bounded by insertion order: a Pending row retires only if its rowid is at or below the high-water mark of rows this instance has served from ReadPendingSinceAsync (clamped on purge, since SQLite reuses rowids); Forwarded rows are exempt (central ACKed them over the push path). At-least-once is unchanged. F5 (LOW) Documented the liveness dependency (a served row never covered by a later cursor stays Pending forever; PurgeExpiredAsync never purges Pending) in ISiteAuditQueue + Component-AuditLog.md, and added a cheap site-health signal: SiteAuditBacklogReporter logs a rate-limited warning when the existing oldest-pending metric exceeds 24h. F6 (MED) SiteAlarmAggregatorActor: _fanoutSinceLastTick was armed by the reconcile's OWN fan-out, so steady state ran fan-out→skip→fan-out→skip — one reconcile per 2x interval (120s), halving the not-reporting refresh and the alarm reconcile backstop. The skip is now armed only by connect/failover-driven seeds (initial, _seedOnConnect, and a re-seed queued behind one). Tests: Communication.Tests 691 passed (+13), AuditLog.Tests 382 passed (+5).
1312 lines
57 KiB
C#
1312 lines
57 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Akka.TestKit.Xunit2;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
|
|
|
/// <summary>
|
|
/// Tests for DebugStreamBridgeActor with gRPC streaming integration.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Shares the <c>DebugStreamStatics</c> xUnit collection with <c>DebugStreamServiceTests</c>:
|
|
/// both tune the actor's static test seams (<c>SnapshotTimeout</c>, <c>ConsumerIdleTimeout</c>,
|
|
/// …), and xUnit parallelizes distinct classes by default, which would let one class's
|
|
/// try/finally restore clobber the other's window mid-test.
|
|
/// </remarks>
|
|
[Collection("DebugStreamStatics")]
|
|
public class DebugStreamBridgeActorTests : TestKit
|
|
{
|
|
private const string SiteId = "site-alpha";
|
|
private const string InstanceName = "Site1.Pump01";
|
|
private const string GrpcNodeA = "http://localhost:5100";
|
|
private const string GrpcNodeB = "http://localhost:5200";
|
|
|
|
public DebugStreamBridgeActorTests() : base(@"akka.loglevel = DEBUG")
|
|
{
|
|
// Use a very short reconnect delay for testing
|
|
DebugStreamBridgeActor.ReconnectDelay = TimeSpan.FromMilliseconds(100);
|
|
// Long stability window so streams are never considered "stable" mid-test
|
|
// unless a test deliberately waits it out.
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
|
}
|
|
|
|
private record TestContext(
|
|
IActorRef BridgeActor,
|
|
TestProbe CommProbe,
|
|
MockSiteStreamGrpcClient MockGrpcClient,
|
|
List<object> ReceivedEvents,
|
|
bool[] TerminatedFlag);
|
|
|
|
private TestContext CreateBridgeActor()
|
|
{
|
|
var commProbe = CreateTestProbe();
|
|
var mockClient = new MockSiteStreamGrpcClient();
|
|
var factory = new MockSiteStreamGrpcClientFactory(mockClient);
|
|
var events = new List<object>();
|
|
var terminated = new[] { false };
|
|
|
|
Action<object> onEvent = evt => { lock (events) { events.Add(evt); } };
|
|
Action onTerminated = () => terminated[0] = true;
|
|
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId,
|
|
InstanceName,
|
|
"corr-1",
|
|
commProbe.Ref,
|
|
onEvent,
|
|
onTerminated,
|
|
factory,
|
|
GrpcNodeA,
|
|
GrpcNodeB);
|
|
|
|
var actor = Sys.ActorOf(props);
|
|
return new TestContext(actor, commProbe, mockClient, events, terminated);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_InstanceNotFound_Snapshot_Forwards_To_OnEvent_Tears_Down_Stream_And_Terminates()
|
|
{
|
|
// M2.11 (revised for M2.18 stream-first): the gRPC subscription is now opened
|
|
// up-front in PreStart, so when the site reports InstanceNotFound=true the
|
|
// bridge actor must
|
|
// (a) forward the not-found snapshot to _onEvent so DebugStreamService's TCS
|
|
// resolves and the caller can inspect the flag,
|
|
// (b) tear DOWN the already-opened gRPC stream (Unsubscribe the just-opened
|
|
// correlation) rather than enter pass-through, and
|
|
// (c) stop itself cleanly.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // initial subscribe envelope
|
|
|
|
// Stream-first: the gRPC subscription is opened before the snapshot arrives.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var notFoundSnapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow,
|
|
InstanceNotFound: true);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(notFoundSnapshot);
|
|
|
|
// (a) _onEvent must receive the not-found snapshot
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
var received = Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.True(received.InstanceNotFound);
|
|
}
|
|
|
|
// (b) the just-opened gRPC stream is torn down (not left running / no pass-through)
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// (c) actor terminates cleanly
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
[Fact]
|
|
public void PreStart_Sends_SubscribeDebugViewRequest_Via_ClusterClient()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.Equal(SiteId, envelope.SiteId);
|
|
Assert.IsType<SubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
var req = (SubscribeDebugViewRequest)envelope.Message;
|
|
Assert.Equal(InstanceName, req.InstanceUniqueName);
|
|
Assert.Equal("corr-1", req.CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Forwards_To_OnEvent_Callback()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Does_Not_Open_Additional_GrpcStream()
|
|
{
|
|
// M2.18 stream-first: the gRPC subscription is opened in PreStart, BEFORE the
|
|
// snapshot arrives. After the snapshot is delivered the actor switches to
|
|
// pass-through — it must NOT open a second subscription. Exactly ONE subscribe
|
|
// call should have been made (the PreStart one).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
// Verify the stream is already open before the snapshot.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// After snapshot delivery, still exactly ONE subscribe — no additional stream opened.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
var singleCall = Assert.Single(ctx.MockGrpcClient.SubscribeCalls);
|
|
Assert.Equal("corr-1", singleCall.CorrelationId);
|
|
Assert.Equal(InstanceName, singleCall.InstanceUniqueName);
|
|
}
|
|
|
|
[Fact]
|
|
public void Events_From_GrpcCallback_Forwarded_To_OnEvent()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC event arriving via the onEvent callback
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(attrChange);
|
|
|
|
// snapshot + attr change
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Reconnects_To_Other_Node()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC error
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Stream broken"));
|
|
|
|
// Should resubscribe
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("corr-1", ctx.MockGrpcClient.SubscribeCalls[1].CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Unsubscribes_Old_Stream_Before_Reconnect()
|
|
{
|
|
// Communication-002 regression: a reconnect must unsubscribe the previous
|
|
// stream so the old node does not keep a zombie relay actor / subscription.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC error → reconnect
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Stream broken"));
|
|
|
|
// Should resubscribe...
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// ...and must have unsubscribed the prior correlation ID so the old node's
|
|
// relay actor is released rather than left zombie.
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
}
|
|
|
|
[Fact]
|
|
public void After_MaxRetries_Terminates()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// 4 consecutive errors: initial + 3 retries, then terminate
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Error 1"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnError(new Exception("Error 2"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 3, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[2].OnError(new Exception("Error 3"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 4, TimeSpan.FromSeconds(5));
|
|
|
|
// Fourth error exceeds max retries
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Error 4"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void StopDebugStream_Cancels_Grpc_And_Sends_Unsubscribe()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
ctx.BridgeActor.Tell(new StopDebugStream());
|
|
|
|
// Should send the site-addressed unsubscribe
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.IsType<UnsubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
// Should unsubscribe gRPC
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Count > 0, TimeSpan.FromSeconds(3));
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
|
|
// Should stop self
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
}
|
|
|
|
[Fact]
|
|
public void DebugStreamTerminated_Stops_Actor_Idempotently()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(new DebugStreamTerminated(SiteId, "corr-1"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void FlappingStream_DeliveringEventsBetweenFailures_StillTerminatesAfterMaxRetries()
|
|
{
|
|
// Communication-008 regression: a stream that connects, delivers an event,
|
|
// then fails — repeatedly — must still trip MaxRetries. The retry count is
|
|
// NO LONGER reset by a received event (only by the stability window). The
|
|
// previous behaviour reset _retryCount on every event, so a flapping site
|
|
// reconnected forever and the debug session lived on indefinitely.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
|
|
// Flap: deliver one event then fail, three times. Each event would, under
|
|
// the old buggy logic, reset the retry budget and prevent termination.
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
var call = ctx.MockGrpcClient.SubscribeCalls[i];
|
|
call.OnEvent(attrChange);
|
|
call.OnError(new Exception($"Flap {i + 1}"));
|
|
var expected = i + 2;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
// Fourth error (after the 3 retries) must exceed MaxRetries and terminate.
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnEvent(attrChange);
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Flap 4"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Reconnects_To_Other_Node_Endpoint()
|
|
{
|
|
// Communication-015 regression: drive the bridge actor through a node flip
|
|
// with an endpoint-aware factory (one distinct mock client per endpoint).
|
|
// The first subscribe targets NodeA; after a gRPC error the bridge must
|
|
// reconnect via a client bound to the *NodeB* endpoint.
|
|
var commProbe = CreateTestProbe();
|
|
var factory = new EndpointTrackingGrpcClientFactory();
|
|
var events = new List<object>();
|
|
var terminated = new[] { false };
|
|
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId, InstanceName, "corr-1", commProbe.Ref,
|
|
(Action<object>)(evt => { lock (events) { events.Add(evt); } }),
|
|
(Action)(() => terminated[0] = true),
|
|
factory, GrpcNodeA, GrpcNodeB);
|
|
|
|
var actor = Sys.ActorOf(props);
|
|
commProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
actor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// First subscribe goes to NodeA.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// gRPC error → bridge flips to NodeB.
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnError(new Exception("NodeA down"));
|
|
|
|
// The reconnect must reach a client bound to the NodeB endpoint.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
Assert.Equal("corr-1", factory.ClientFor(GrpcNodeB).SubscribeCalls[0].CorrelationId);
|
|
}
|
|
|
|
// ── WP1.1: graceful (status OK) stream completion is a reconnect trigger ──
|
|
|
|
[Fact]
|
|
public void On_GracefulStreamCompletion_Reopens_On_The_Same_Node()
|
|
{
|
|
// The site caps every stream at GrpcMaxStreamLifetime and then ends the RPC with
|
|
// OK. Pre-fix the client's read loop just finished, the actor was told nothing, and
|
|
// the debug session went silently deaf for the rest of its life.
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnCompleted();
|
|
|
|
// Reopened on the SAME node — a clean close is not a fault, so there is nothing to
|
|
// fail over from — and the finished stream is released, not left zombie.
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 2,
|
|
TimeSpan.FromSeconds(5));
|
|
Assert.Empty(factory.ClientFor(GrpcNodeB).SubscribeCalls);
|
|
Assert.Contains("corr-1", factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds);
|
|
}
|
|
|
|
[Fact]
|
|
public void RepeatedGracefulCompletions_DoNotConsume_TheErrorRetryBudget()
|
|
{
|
|
// Five completions — two more than MaxRetries. Had completion been routed through
|
|
// the error path the session would have terminated on the fourth.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
for (var i = 1; i <= 5; i++)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[i - 1].OnCompleted();
|
|
var expected = i + 1;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected,
|
|
TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void LateCompletionFromAPreviousStreamGeneration_IsIgnored()
|
|
{
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
var firstSub = factory.ClientFor(GrpcNodeA).SubscribeCalls[0];
|
|
firstSub.OnError(new Exception("NodeA down")); // gen 1 dies → gen 2 opens on NodeB
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
|
|
// A completion racing out of the dead gen-1 stream must not tear down the live one.
|
|
firstSub.OnCompleted();
|
|
Thread.Sleep(300);
|
|
Assert.DoesNotContain("corr-1", factory.ClientFor(GrpcNodeB).UnsubscribedCorrelationIds);
|
|
Assert.Single(factory.ClientFor(GrpcNodeB).SubscribeCalls);
|
|
}
|
|
|
|
// ── Task 6 (arch review 02, High): teardown/failover unsubscribe is endpoint-safe ──
|
|
// Both paths use TryGet, never GetOrCreate, so cleanup can never open a fresh
|
|
// channel or (with (site,endpoint) keying) touch another session's channel.
|
|
|
|
private (IActorRef Actor, EndpointTrackingGrpcClientFactory Factory) CreateBridgeWithTrackingFactory()
|
|
{
|
|
var commProbe = CreateTestProbe();
|
|
var factory = new EndpointTrackingGrpcClientFactory();
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId, InstanceName, "corr-1", commProbe.Ref,
|
|
(Action<object>)(_ => { }), (Action)(() => { }),
|
|
factory, GrpcNodeA, GrpcNodeB);
|
|
var actor = Sys.ActorOf(props);
|
|
commProbe.ExpectMsg<SiteEnvelope>(); // PreStart subscribe handshake
|
|
return (actor, factory);
|
|
}
|
|
|
|
[Fact]
|
|
public void CleanupGrpc_UnsubscribesViaTryGet_WithoutOpeningAChannel()
|
|
{
|
|
var (actor, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
var createdBefore = factory.CreatedCount; // node-a only
|
|
|
|
actor.Tell(new StopDebugStream());
|
|
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
// Pre-fix CleanupGrpc called GetOrCreate → could open a fresh channel here.
|
|
Assert.Equal(createdBefore, factory.CreatedCount);
|
|
}
|
|
|
|
[Fact]
|
|
public void SessionFailover_UnsubscribesFailedEndpointViaTryGet_OpensOnlyTheReconnectChannel()
|
|
{
|
|
var (_, factory) = CreateBridgeWithTrackingFactory();
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Fail NodeA: the bridge unsubscribes the failed NodeA stream via TryGet
|
|
// (if that path had used GetOrCreate on the post-flip cache it could have
|
|
// disposed/created the wrong client), then reconnects to NodeB.
|
|
factory.ClientFor(GrpcNodeA).SubscribeCalls[0].OnError(new Exception("NodeA down"));
|
|
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeA).UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(5));
|
|
AwaitCondition(() => factory.ClientFor(GrpcNodeB).SubscribeCalls.Count == 1,
|
|
TimeSpan.FromSeconds(5));
|
|
// Only the two real node endpoints were ever opened — the unsubscribe
|
|
// itself created nothing.
|
|
Assert.Equal(2, factory.CreatedCount);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------
|
|
// M2.18 (#26) — stream-first + replay/dedup
|
|
// ---------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void PreStart_Opens_GrpcStream_Before_Snapshot_Arrives()
|
|
{
|
|
// M2.18: the gRPC subscription must be opened in PreStart (stream-first),
|
|
// BEFORE the snapshot is delivered, so live events start flowing during the
|
|
// snapshot-build + network-transit window. The old lifecycle opened the
|
|
// stream only after the snapshot arrived, losing gap-window events.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // initial subscribe envelope
|
|
|
|
// No snapshot sent yet — the stream must already be open.
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Assert.Equal("corr-1", ctx.MockGrpcClient.SubscribeCalls[0].CorrelationId);
|
|
Assert.Equal(InstanceName, ctx.MockGrpcClient.SubscribeCalls[0].InstanceUniqueName);
|
|
|
|
// _onEvent must NOT have fired — buffering, not delivering.
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
}
|
|
|
|
[Fact]
|
|
public void GapWindow_Event_Buffered_Before_Snapshot_Is_Delivered_Exactly_Once_After_Snapshot()
|
|
{
|
|
// M2.18: an event arriving DURING the snapshot window (before the snapshot
|
|
// is delivered) is buffered, then flushed exactly once AFTER the snapshot.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Live event arrives BEFORE the snapshot — its entity is NOT in the snapshot,
|
|
// so it is a genuine gap-window event that must survive.
|
|
var gapEvent = new AttributeValueChanged(InstanceName, "IO", "Pressure", 99.9, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(gapEvent);
|
|
|
|
// While buffering, _onEvent has not fired.
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot then the buffered gap-window event, exactly once, in that order.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal("Pressure", flushed.AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Event_Already_Reflected_In_Snapshot_Is_Dropped()
|
|
{
|
|
// M2.18 dedup: a buffered event whose entity is in the snapshot with an equal
|
|
// or newer snapshot timestamp (buffered.Timestamp <= snapshot.Timestamp) is
|
|
// already reflected and must be DROPPED.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var t0 = DateTimeOffset.UtcNow;
|
|
|
|
// Buffered event for "Temp" at t0.
|
|
var buffered = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", t0);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(buffered);
|
|
|
|
// Snapshot already contains "Temp" at the SAME timestamp t0 → buffered is a dup.
|
|
var snapAttr = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", t0);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged> { snapAttr },
|
|
new List<AlarmStateChanged>(),
|
|
t0);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// Only the snapshot is delivered; the buffered duplicate is dropped.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
// Give a beat to ensure no extra (dropped) event sneaks through.
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Single(ctx.ReceivedEvents);
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Event_Strictly_Newer_Than_Snapshot_Entity_Is_Delivered()
|
|
{
|
|
// M2.18 dedup: a buffered event strictly newer than the snapshot's entry for
|
|
// the same entity (buffered.Timestamp > snapshot.Timestamp) is NOT a dup and
|
|
// must be DELIVERED after the snapshot.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapTime = DateTimeOffset.UtcNow;
|
|
var newerTime = snapTime.AddMilliseconds(1);
|
|
|
|
// Buffered event for "Temp" strictly NEWER than the snapshot's "Temp".
|
|
var buffered = new AttributeValueChanged(InstanceName, "IO", "Temp", 50.0, "Good", newerTime);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(buffered);
|
|
|
|
var snapAttr = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", snapTime);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged> { snapAttr },
|
|
new List<AlarmStateChanged>(),
|
|
snapTime);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot then the strictly-newer buffered event.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal(50.0, flushed.Value);
|
|
Assert.Equal(newerTime, flushed.Timestamp);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Alarm_Dedup_Uses_AlarmIdentity_And_Timestamp()
|
|
{
|
|
// M2.18 dedup for alarms: identity = (instance, alarm name, source reference).
|
|
// A buffered alarm older-or-equal to the snapshot's same-identity alarm is
|
|
// dropped; a strictly-newer one is delivered.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var t0 = DateTimeOffset.UtcNow;
|
|
|
|
// Buffered: "PumpFault" at t0 (dup) and "Overheat" at t0+1ms (newer, deliver).
|
|
var dupAlarm = new AlarmStateChanged(InstanceName, "PumpFault",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 500, t0);
|
|
var newerAlarm = new AlarmStateChanged(InstanceName, "Overheat",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 700, t0.AddMilliseconds(1));
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(dupAlarm);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(newerAlarm);
|
|
|
|
// Snapshot contains BOTH "PumpFault" and "Overheat" at t0.
|
|
var snapPumpFault = new AlarmStateChanged(InstanceName, "PumpFault",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 500, t0);
|
|
var snapOverheat = new AlarmStateChanged(InstanceName, "Overheat",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Normal, 0, t0);
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged> { snapPumpFault, snapOverheat },
|
|
t0);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot + only the strictly-newer "Overheat" alarm (PumpFault dropped).
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Equal(2, ctx.ReceivedEvents.Count);
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
var flushed = Assert.IsType<AlarmStateChanged>(ctx.ReceivedEvents[1]);
|
|
Assert.Equal("Overheat", flushed.AlarmName);
|
|
Assert.Equal(700, flushed.Priority);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Buffered_Events_Flushed_In_Arrival_Order()
|
|
{
|
|
// M2.18: ordering preserved across multiple buffered events (none are dups —
|
|
// their entities are absent from the snapshot).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var baseTime = DateTimeOffset.UtcNow;
|
|
var sub = ctx.MockGrpcClient.SubscribeCalls[0];
|
|
sub.OnEvent(new AttributeValueChanged(InstanceName, "IO", "A", 1, "Good", baseTime));
|
|
sub.OnEvent(new AlarmStateChanged(InstanceName, "AlarmX",
|
|
ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AlarmState.Active, 100, baseTime));
|
|
sub.OnEvent(new AttributeValueChanged(InstanceName, "IO", "B", 2, "Good", baseTime));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
baseTime);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 4; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.Equal("A", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
Assert.Equal("AlarmX", Assert.IsType<AlarmStateChanged>(ctx.ReceivedEvents[2]).AlarmName);
|
|
Assert.Equal("B", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[3]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void PassThrough_After_Flush_Delivers_Subsequent_Events_Immediately()
|
|
{
|
|
// M2.18: after the snapshot+flush the actor switches to pass-through — later
|
|
// events go straight to _onEvent (no buffering, no dup).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Post-snapshot event — must be delivered immediately, exactly once.
|
|
var postEvent = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(postEvent);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void InstanceNotFound_After_StreamFirst_Tears_Down_Stream_And_Does_Not_PassThrough()
|
|
{
|
|
// M2.18 + M2.11: stream-first means the gRPC subscription is already open
|
|
// when an InstanceNotFound snapshot arrives. The bridge must tear that stream
|
|
// down (Unsubscribe the just-opened correlation), deliver the not-found
|
|
// snapshot, NOT enter pass-through, and stop cleanly.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
// Stream opened up-front (stream-first).
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var notFoundSnapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow,
|
|
InstanceNotFound: true);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(notFoundSnapshot);
|
|
|
|
// Not-found snapshot delivered.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.True(Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]).InstanceNotFound);
|
|
}
|
|
|
|
// The just-opened stream must be torn down.
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Stops cleanly.
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
|
|
// No pass-through: an event arriving after the stop is not delivered.
|
|
var late = new AttributeValueChanged(InstanceName, "IO", "Temp", 1, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(late);
|
|
Thread.Sleep(200);
|
|
lock (ctx.ReceivedEvents) { Assert.Single(ctx.ReceivedEvents); }
|
|
}
|
|
|
|
[Fact]
|
|
public void Reconnect_During_Buffering_Phase_Keeps_Buffering_Until_Snapshot()
|
|
{
|
|
// M2.18: a gRPC error/reconnect BEFORE the snapshot arrives must remain in the
|
|
// buffering phase — events on the new stream are still buffered, then flushed
|
|
// when the snapshot finally arrives.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Error before snapshot → reconnect (still buffering).
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("pre-snapshot blip"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Event on the reconnected stream — still buffered (snapshot not yet delivered).
|
|
var gapEvent = new AttributeValueChanged(InstanceName, "IO", "Late", 7, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnEvent(gapEvent);
|
|
lock (ctx.ReceivedEvents) { Assert.Empty(ctx.ReceivedEvents); }
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
// snapshot + the event buffered across the reconnect.
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]);
|
|
Assert.Equal("Late", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Reconnect_After_Snapshot_Resumes_PassThrough_Not_Buffering()
|
|
{
|
|
// M2.18: a mid-session reconnect (after the snapshot was already delivered)
|
|
// must resume pass-through — the snapshot is a one-time thing and events on
|
|
// the reconnected stream are delivered immediately, not re-buffered.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Mid-session reconnect.
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("mid-session blip"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Event on the reconnected stream — delivered immediately (pass-through).
|
|
var postEvent = new AttributeValueChanged(InstanceName, "IO", "Temp", 9, "Good",
|
|
DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnEvent(postEvent);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
Assert.Equal("Temp", Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]).AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void RetryCount_RecoveredOnlyAfterStreamStaysStableForStabilityWindow()
|
|
{
|
|
// Communication-008: after a stream has been connected for the stability
|
|
// window, the retry budget is recovered — a later transient fault then gets
|
|
// a fresh set of retries rather than being counted against the old budget.
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Two failures — but each new stream stays up long enough (the mock
|
|
// stream only completes on cancel) for the stability window to elapse
|
|
// and reset the retry budget before the next failure.
|
|
for (var i = 0; i < 5; i++)
|
|
{
|
|
Thread.Sleep(450); // exceed the 300ms stability window
|
|
ctx.MockGrpcClient.SubscribeCalls[i].OnError(new Exception($"Error {i + 1}"));
|
|
var expected = i + 2;
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == expected, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
// Five well-spaced failures did NOT terminate the actor because each
|
|
// reconnect recovered its retry budget after the stability window.
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.StabilityWindow = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
// ── WP2.3: bounded pre-snapshot buffer, hard snapshot deadline, timeout hygiene ──
|
|
|
|
[Fact]
|
|
public void PreSnapshotBuffer_IsCapped_DropsOldest_AndCountsTheDrops()
|
|
{
|
|
// Before the cap a snapshot that never arrived buffered every live event on the
|
|
// CENTRAL node without limit — one wedged session on a chatty instance was enough
|
|
// to grow unbounded. Now the oldest are evicted and counted.
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe envelope; no snapshot is ever sent
|
|
|
|
const int cap = 20_000;
|
|
const int overflow = 250;
|
|
var before = Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped);
|
|
|
|
var t = DateTimeOffset.UtcNow;
|
|
for (var i = 0; i < cap + overflow; i++)
|
|
{
|
|
ctx.BridgeActor.Tell(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", $"Attr{i}", i, "Good", t.AddMilliseconds(i)));
|
|
}
|
|
|
|
// The overflow was evicted (drop-oldest) and counted.
|
|
AwaitCondition(
|
|
() => Interlocked.Read(ref DebugStreamBridgeActor.TotalPreSnapshotDropped) - before >= overflow,
|
|
TimeSpan.FromSeconds(10));
|
|
|
|
// The session is still healthy: the snapshot can still arrive and flush the
|
|
// (capped) buffer — the newest events, which the snapshot may predate, survived.
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
t.AddMilliseconds(-1));
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count >= cap; }
|
|
}, TimeSpan.FromSeconds(10));
|
|
|
|
lock (ctx.ReceivedEvents)
|
|
{
|
|
// Snapshot + exactly the retained (capped) events, and the newest survived.
|
|
Assert.Equal(cap + 1, ctx.ReceivedEvents.Count);
|
|
var lastAttr = ctx.ReceivedEvents.OfType<AttributeValueChanged>().Last();
|
|
Assert.Equal($"Attr{cap + overflow - 1}", lastAttr.AttributeName);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void NoSnapshotWithinDeadline_FailsTheSession_InsteadOfBufferingForever()
|
|
{
|
|
// Nothing else ends a session wedged in the buffering phase: a lost site reply
|
|
// raises no gRPC error, and stream events no longer reset the orphan timeout.
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe request — never answered
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
|
|
// The consumer is told, so the UI can surface the failure and reopen.
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
// And the site-side relay was released rather than left as a zombie.
|
|
AwaitCondition(
|
|
() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Contains("corr-1"),
|
|
TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void SnapshotArrival_StandsDownTheDeadline()
|
|
{
|
|
// The deadline must not fire after a healthy snapshot — a live session would
|
|
// otherwise be killed mid-stream.
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromMilliseconds(300);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
Thread.Sleep(700); // well past the deadline
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// Still serving: a post-snapshot event passes straight through.
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AlarmStateChanged(
|
|
InstanceName, "PumpFault", Commons.Types.Enums.AlarmState.Active, 500,
|
|
DateTimeOffset.UtcNow));
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AlarmStateChanged>().Any(); }
|
|
}, TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.SnapshotTimeout = TimeSpan.FromSeconds(60);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void StreamEvents_StillReachTheConsumer_ThroughTheWrappedCallbackPath()
|
|
{
|
|
// The gRPC callback wraps every event in LiveDebugStreamEvent; the wrapper must be
|
|
// transparent to delivery (it exists only to keep the high-volume path explicit —
|
|
// the orphan net keys off the consumer keepalive, not the mailbox).
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
var evt = new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(evt);
|
|
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Any(); }
|
|
}, TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
// ----- Orphan net: measures the CONSUMER, not the mailbox ----- //
|
|
|
|
[Fact]
|
|
public void HealthySession_StreamingEvents_WithConsumerKeepalives_SurvivesWellPastTheOrphanWindow()
|
|
{
|
|
// THE regression this closes: with the orphan net armed off the mailbox
|
|
// (SetReceiveTimeout) and stream events correctly excluded from it, nothing recurring
|
|
// reset it — the snapshot lands once, GrpcStreamStable once — so a perfectly healthy
|
|
// session self-terminated one window later and the consumer was told
|
|
// "Site disconnected". Here the session streams events and receives the keepalive
|
|
// DebugStreamService sends while it is attached; it must live through MANY windows.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Watch(ctx.BridgeActor);
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// ~5 orphan windows of pure stream traffic + consumer keepalives, and no other
|
|
// mailbox activity whatsoever (no reconnects, no snapshots, no stop).
|
|
var deadline = DateTime.UtcNow.AddSeconds(2);
|
|
var delivered = 0;
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 20.0 + delivered, "Good",
|
|
DateTimeOffset.UtcNow));
|
|
delivered++;
|
|
// What DebugStreamService's shared timer does for an attached session.
|
|
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive());
|
|
Thread.Sleep(100);
|
|
}
|
|
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(100));
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// Still serving: the events all arrived and the actor is alive.
|
|
AwaitCondition(() =>
|
|
{
|
|
lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.OfType<AttributeValueChanged>().Count() == delivered; }
|
|
}, TimeSpan.FromSeconds(3));
|
|
Assert.False(ctx.BridgeActor.IsNobody());
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void OrphanedSession_ConsumerGone_StillTerminates_EvenWhileEventsKeepArriving()
|
|
{
|
|
// The other half of the contract: site chatter must NOT hold an abandoned session
|
|
// open. No keepalive arrives (the consumer is gone), so the session terminates,
|
|
// unsubscribes from the site and reports termination — while events stream in.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(400);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
Watch(ctx.BridgeActor);
|
|
|
|
ctx.BridgeActor.Tell(new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow));
|
|
|
|
// Keep the site chatty for longer than the orphan window — with no keepalive.
|
|
var deadline = DateTime.UtcNow.AddSeconds(1);
|
|
while (DateTime.UtcNow < deadline)
|
|
{
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(new AttributeValueChanged(
|
|
InstanceName, "Modules.IO", "Temperature", 42.5, "Good", DateTimeOffset.UtcNow));
|
|
Thread.Sleep(50);
|
|
}
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void ConsumerKeepalive_RenewsTheWindow_AfterANearMiss()
|
|
{
|
|
// A single late keepalive is enough to save a session — the stamp is refreshed, not
|
|
// a one-shot grace period.
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMilliseconds(500);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromMilliseconds(50);
|
|
try
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Watch(ctx.BridgeActor);
|
|
|
|
Thread.Sleep(350); // most of the window burnt
|
|
ctx.BridgeActor.Tell(new DebugStreamConsumerAlive()); // …then the consumer checks in
|
|
Thread.Sleep(350); // past the ORIGINAL deadline
|
|
|
|
Assert.False(ctx.TerminatedFlag[0]);
|
|
|
|
// …and once the keepalives stop, it does terminate.
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(3));
|
|
}
|
|
finally
|
|
{
|
|
DebugStreamBridgeActor.ConsumerIdleTimeout = TimeSpan.FromMinutes(5);
|
|
DebugStreamBridgeActor.ConsumerLivenessCheckInterval = TimeSpan.FromSeconds(30);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Mock gRPC client that records SubscribeAsync and Unsubscribe calls.
|
|
/// <para>
|
|
/// <b>Thread safety:</b> <see cref="SubscribeCalls"/> and
|
|
/// <see cref="UnsubscribedCorrelationIds"/> are written from the actor/background thread
|
|
/// (via <see cref="SubscribeAsync"/> and <see cref="Unsubscribe"/>) and read from the test
|
|
/// thread (via <c>AwaitCondition</c> / assertions). All access goes through a shared lock
|
|
/// to match the <c>lock (events)</c> pattern used for <c>ctx.ReceivedEvents</c>.
|
|
/// </para>
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClient : SiteStreamGrpcClient
|
|
{
|
|
private readonly object _lock = new();
|
|
private readonly List<MockSubscription> _subscribeCalls = new();
|
|
private readonly List<string> _unsubscribedCorrelationIds = new();
|
|
|
|
/// <summary>Returns a snapshot of subscribe calls, taken under the internal lock.</summary>
|
|
public List<MockSubscription> SubscribeCalls { get { lock (_lock) { return _subscribeCalls.ToList(); } } }
|
|
|
|
/// <summary>Returns a snapshot of unsubscribed correlation IDs, taken under the internal lock.</summary>
|
|
public List<string> UnsubscribedCorrelationIds { get { lock (_lock) { return _unsubscribedCorrelationIds.ToList(); } } }
|
|
|
|
private MockSiteStreamGrpcClient(bool _) : base() { }
|
|
|
|
public MockSiteStreamGrpcClient() : base()
|
|
{
|
|
}
|
|
|
|
public override Task SubscribeAsync(
|
|
string correlationId,
|
|
string instanceUniqueName,
|
|
Action<object> onEvent,
|
|
Action<Exception> onError,
|
|
Action onCompleted,
|
|
CancellationToken ct)
|
|
{
|
|
var subscription = new MockSubscription(
|
|
correlationId, instanceUniqueName, onEvent, onError, onCompleted, ct);
|
|
lock (_lock) { _subscribeCalls.Add(subscription); }
|
|
|
|
// Return a task that completes when cancelled (simulates long-running stream)
|
|
var tcs = new TaskCompletionSource();
|
|
ct.Register(() => tcs.TrySetResult());
|
|
return tcs.Task;
|
|
}
|
|
|
|
public override void Unsubscribe(string correlationId)
|
|
{
|
|
lock (_lock) { _unsubscribedCorrelationIds.Add(correlationId); }
|
|
}
|
|
}
|
|
|
|
internal record MockSubscription(
|
|
string CorrelationId,
|
|
string InstanceUniqueName,
|
|
Action<object> OnEvent,
|
|
Action<Exception> OnError,
|
|
Action OnCompleted,
|
|
CancellationToken CancellationToken);
|
|
|
|
/// <summary>
|
|
/// Factory that always returns the pre-configured mock client.
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClientFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly MockSiteStreamGrpcClient _mockClient;
|
|
public List<string> RequestedEndpoints { get; } = new();
|
|
|
|
public MockSiteStreamGrpcClientFactory(MockSiteStreamGrpcClient mockClient)
|
|
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance)
|
|
{
|
|
_mockClient = mockClient;
|
|
}
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
{
|
|
RequestedEndpoints.Add(grpcEndpoint);
|
|
return _mockClient;
|
|
}
|
|
|
|
// Mirrors the real factory's TryGet: returns the client only for an endpoint
|
|
// already opened via GetOrCreate, never creating one. Lets teardown/failover
|
|
// unsubscribe paths (now TryGet-based, Task 6) resolve the mock client.
|
|
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
|
=> RequestedEndpoints.Contains(grpcEndpoint) ? _mockClient : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Endpoint-aware mock factory: hands out a distinct <see cref="MockSiteStreamGrpcClient"/>
|
|
/// per endpoint, mirroring the real factory's corrected NodeA→NodeB failover behaviour
|
|
/// so node-flip coverage is meaningful (Communication-015).
|
|
/// </summary>
|
|
internal class EndpointTrackingGrpcClientFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly System.Collections.Concurrent.ConcurrentDictionary<string, MockSiteStreamGrpcClient> _byEndpoint = new();
|
|
|
|
public EndpointTrackingGrpcClientFactory()
|
|
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance)
|
|
{
|
|
}
|
|
|
|
public MockSiteStreamGrpcClient ClientFor(string endpoint) =>
|
|
_byEndpoint.GetOrAdd(endpoint, _ => new MockSiteStreamGrpcClient());
|
|
|
|
/// <summary>Number of distinct endpoint clients created so far (channels opened).</summary>
|
|
public int CreatedCount => _byEndpoint.Count;
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
=> ClientFor(grpcEndpoint);
|
|
|
|
// TryGet never creates: returns the client only for an already-opened endpoint.
|
|
public override SiteStreamGrpcClient? TryGet(string siteIdentifier, string grpcEndpoint)
|
|
=> _byEndpoint.TryGetValue(grpcEndpoint, out var client) ? client : null;
|
|
}
|