After receiving the initial snapshot via ClusterClient, the bridge actor now opens a gRPC server-streaming subscription via SiteStreamGrpcClient for ongoing AttributeValueChanged/AlarmStateChanged events. Adds NodeA/ NodeB failover with max 3 retries, retry count reset on successful event, and IWithTimers-based reconnect scheduling. - DebugStreamBridgeActor: gRPC stream after snapshot, reconnect state machine - DebugStreamService: inject SiteStreamGrpcClientFactory, resolve gRPC addresses - ServiceCollectionExtensions: register SiteStreamGrpcClientFactory singleton - SiteStreamGrpcClient: make SubscribeAsync/Unsubscribe virtual for testability - SiteStreamGrpcClientFactory: make GetOrCreate virtual for testability - New test suite: DebugStreamBridgeActorTests (8 tests)
340 lines
12 KiB
C#
340 lines
12 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit;
|
|
using Akka.TestKit.Xunit2;
|
|
using ScadaLink.Commons.Messages.DebugView;
|
|
using ScadaLink.Commons.Messages.Streaming;
|
|
using ScadaLink.Communication.Actors;
|
|
using ScadaLink.Communication.Grpc;
|
|
|
|
namespace ScadaLink.Communication.Tests.Grpc;
|
|
|
|
/// <summary>
|
|
/// Tests for DebugStreamBridgeActor with gRPC streaming integration.
|
|
/// </summary>
|
|
public class DebugStreamBridgeActorTests : TestKit
|
|
{
|
|
private const string SiteId = "site-alpha";
|
|
private const string InstanceName = "Site1.Pump01";
|
|
private const string GrpcNodeA = "http://localhost:5100";
|
|
private const string GrpcNodeB = "http://localhost:5200";
|
|
|
|
public DebugStreamBridgeActorTests() : base(@"akka.loglevel = DEBUG")
|
|
{
|
|
// Use a very short reconnect delay for testing
|
|
DebugStreamBridgeActor.ReconnectDelay = TimeSpan.FromMilliseconds(100);
|
|
}
|
|
|
|
private record TestContext(
|
|
IActorRef BridgeActor,
|
|
TestProbe CommProbe,
|
|
MockSiteStreamGrpcClient MockGrpcClient,
|
|
List<object> ReceivedEvents,
|
|
bool[] TerminatedFlag);
|
|
|
|
private TestContext CreateBridgeActor()
|
|
{
|
|
var commProbe = CreateTestProbe();
|
|
var mockClient = new MockSiteStreamGrpcClient();
|
|
var factory = new MockSiteStreamGrpcClientFactory(mockClient);
|
|
var events = new List<object>();
|
|
var terminated = new[] { false };
|
|
|
|
Action<object> onEvent = evt => { lock (events) { events.Add(evt); } };
|
|
Action onTerminated = () => terminated[0] = true;
|
|
|
|
var props = Props.Create(typeof(DebugStreamBridgeActor),
|
|
SiteId,
|
|
InstanceName,
|
|
"corr-1",
|
|
commProbe.Ref,
|
|
onEvent,
|
|
onTerminated,
|
|
factory,
|
|
GrpcNodeA,
|
|
GrpcNodeB);
|
|
|
|
var actor = Sys.ActorOf(props);
|
|
return new TestContext(actor, commProbe, mockClient, events, terminated);
|
|
}
|
|
|
|
[Fact]
|
|
public void PreStart_Sends_SubscribeDebugViewRequest_Via_ClusterClient()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.Equal(SiteId, envelope.SiteId);
|
|
Assert.IsType<SubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
var req = (SubscribeDebugViewRequest)envelope.Message;
|
|
Assert.Equal(InstanceName, req.InstanceUniqueName);
|
|
Assert.Equal("corr-1", req.CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Forwards_To_OnEvent_Callback()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 1; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<DebugViewSnapshot>(ctx.ReceivedEvents[0]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_Snapshot_Opens_GrpcStream()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
var call = ctx.MockGrpcClient.SubscribeCalls[0];
|
|
Assert.Equal("corr-1", call.CorrelationId);
|
|
Assert.Equal(InstanceName, call.InstanceUniqueName);
|
|
}
|
|
|
|
[Fact]
|
|
public void Events_From_GrpcCallback_Forwarded_To_OnEvent()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC event arriving via the onEvent callback
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnEvent(attrChange);
|
|
|
|
// snapshot + attr change
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
lock (ctx.ReceivedEvents) { Assert.IsType<AttributeValueChanged>(ctx.ReceivedEvents[1]); }
|
|
}
|
|
|
|
[Fact]
|
|
public void On_GrpcError_Reconnects_To_Other_Node()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// Simulate gRPC error
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Stream broken"));
|
|
|
|
// Should resubscribe
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
Assert.Equal("corr-1", ctx.MockGrpcClient.SubscribeCalls[1].CorrelationId);
|
|
}
|
|
|
|
[Fact]
|
|
public void After_MaxRetries_Terminates()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// 4 consecutive errors: initial + 3 retries, then terminate
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Error 1"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnError(new Exception("Error 2"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 3, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[2].OnError(new Exception("Error 3"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 4, TimeSpan.FromSeconds(5));
|
|
|
|
// Fourth error exceeds max retries
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Error 4"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor, TimeSpan.FromSeconds(5));
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void StopDebugStream_Cancels_Grpc_And_Sends_Unsubscribe()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>(); // subscribe
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
ctx.BridgeActor.Tell(new StopDebugStream());
|
|
|
|
// Should send ClusterClient unsubscribe
|
|
var envelope = ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
Assert.IsType<UnsubscribeDebugViewRequest>(envelope.Message);
|
|
|
|
// Should unsubscribe gRPC
|
|
AwaitCondition(() => ctx.MockGrpcClient.UnsubscribedCorrelationIds.Count > 0, TimeSpan.FromSeconds(3));
|
|
Assert.Contains("corr-1", ctx.MockGrpcClient.UnsubscribedCorrelationIds);
|
|
|
|
// Should stop self
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
}
|
|
|
|
[Fact]
|
|
public void DebugStreamTerminated_Stops_Actor_Idempotently()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
Watch(ctx.BridgeActor);
|
|
ctx.BridgeActor.Tell(new DebugStreamTerminated(SiteId, "corr-1"));
|
|
|
|
ExpectTerminated(ctx.BridgeActor);
|
|
Assert.True(ctx.TerminatedFlag[0]);
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_Error_Resets_RetryCount_On_Successful_Event()
|
|
{
|
|
var ctx = CreateBridgeActor();
|
|
ctx.CommProbe.ExpectMsg<SiteEnvelope>();
|
|
|
|
var snapshot = new DebugViewSnapshot(
|
|
InstanceName,
|
|
new List<AttributeValueChanged>(),
|
|
new List<AlarmStateChanged>(),
|
|
DateTimeOffset.UtcNow);
|
|
|
|
ctx.BridgeActor.Tell(snapshot);
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 1, TimeSpan.FromSeconds(3));
|
|
|
|
// First error → retry 1
|
|
ctx.MockGrpcClient.SubscribeCalls[0].OnError(new Exception("Error 1"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Simulate successful event (resets retry count)
|
|
var attrChange = new AttributeValueChanged(InstanceName, "IO", "Temp", 42.5, "Good", DateTimeOffset.UtcNow);
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnEvent(attrChange);
|
|
AwaitCondition(() => { lock (ctx.ReceivedEvents) { return ctx.ReceivedEvents.Count == 2; } },
|
|
TimeSpan.FromSeconds(3));
|
|
|
|
// Now another 3 errors should be tolerated (retry count was reset)
|
|
ctx.MockGrpcClient.SubscribeCalls[1].OnError(new Exception("Error 2"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 3, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[2].OnError(new Exception("Error 3"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 4, TimeSpan.FromSeconds(5));
|
|
|
|
ctx.MockGrpcClient.SubscribeCalls[3].OnError(new Exception("Error 4"));
|
|
AwaitCondition(() => ctx.MockGrpcClient.SubscribeCalls.Count == 5, TimeSpan.FromSeconds(5));
|
|
|
|
// Still alive — 3 retries from the second failure point succeeded
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Mock gRPC client that records SubscribeAsync and Unsubscribe calls.
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClient : SiteStreamGrpcClient
|
|
{
|
|
public List<MockSubscription> SubscribeCalls { get; } = new();
|
|
public List<string> UnsubscribedCorrelationIds { get; } = new();
|
|
|
|
private MockSiteStreamGrpcClient(bool _) : base() { }
|
|
|
|
public MockSiteStreamGrpcClient() : base()
|
|
{
|
|
}
|
|
|
|
public override Task SubscribeAsync(
|
|
string correlationId,
|
|
string instanceUniqueName,
|
|
Action<object> onEvent,
|
|
Action<Exception> onError,
|
|
CancellationToken ct)
|
|
{
|
|
var subscription = new MockSubscription(correlationId, instanceUniqueName, onEvent, onError, ct);
|
|
SubscribeCalls.Add(subscription);
|
|
|
|
// Return a task that completes when cancelled (simulates long-running stream)
|
|
var tcs = new TaskCompletionSource();
|
|
ct.Register(() => tcs.TrySetResult());
|
|
return tcs.Task;
|
|
}
|
|
|
|
public override void Unsubscribe(string correlationId)
|
|
{
|
|
UnsubscribedCorrelationIds.Add(correlationId);
|
|
}
|
|
}
|
|
|
|
internal record MockSubscription(
|
|
string CorrelationId,
|
|
string InstanceUniqueName,
|
|
Action<object> OnEvent,
|
|
Action<Exception> OnError,
|
|
CancellationToken CancellationToken);
|
|
|
|
/// <summary>
|
|
/// Factory that always returns the pre-configured mock client.
|
|
/// </summary>
|
|
internal class MockSiteStreamGrpcClientFactory : SiteStreamGrpcClientFactory
|
|
{
|
|
private readonly MockSiteStreamGrpcClient _mockClient;
|
|
public List<string> RequestedEndpoints { get; } = new();
|
|
|
|
public MockSiteStreamGrpcClientFactory(MockSiteStreamGrpcClient mockClient)
|
|
: base(Microsoft.Extensions.Logging.Abstractions.NullLoggerFactory.Instance)
|
|
{
|
|
_mockClient = mockClient;
|
|
}
|
|
|
|
public override SiteStreamGrpcClient GetOrCreate(string siteIdentifier, string grpcEndpoint)
|
|
{
|
|
RequestedEndpoints.Add(grpcEndpoint);
|
|
return _mockClient;
|
|
}
|
|
}
|