feat(mesh): gRPC ApplyAck path behind Mode=Grpc
Task 1B.2 — the node→central half of the ClusterClient→gRPC migration. GrpcApplyAckSender walks MeshTransport:CentralGrpcEndpoints in order and STOPS AT THE FIRST SUCCESS. That single `return` is the whole single-count guarantee: the transport it replaces was ClusterClient.Send, deliberately not SendToAll, because both central nodes forward to the SAME ConfigPublishCoordinator singleton proxy — an ack posted to both would have the coordinator count one node twice and could seal a deployment on a doubled count. The list is an availability list, not a fan-out list. No retry beyond that one ordered pass, no queue: the receptionist buffered nothing and retried nothing, and this is like-for-like, not an upgrade to delivery guarantees. All endpoints failing is still a DROP — but a loud one, naming the node, deployment, correlation id and the status each endpoint answered with, which is what separates "central is down" from "central rejected this node's key". The bearer never reaches a log; only endpoints and status codes do. Per-endpoint GrpcChannel cache and the endpoint loop clone GrpcDeploymentArtifactFetcher; the cluster id in x-otopcua-cluster comes from this node's own cluster-<id> role via RoleParser, off the same bound Cluster:Roles ClusterRoleInfo reads. SendAsync never throws: the caller discards the task, so an escaping exception would become an unobserved fault — a drop with no log line, the exact failure mode this migration removes. NodeCommunicationActor gains an OPTIONAL ackSender; every existing test compiles and passes unedited. It is checked before the ClusterClient so the two can never both fire. The registration arm creates no ClusterClient under Grpc. NOTE — one wiring change beyond the actor: DriverHostActor's ackRouter is now `useClusterClient || useGrpc ? nodeComm : null`. Left at ClusterClient-only, a Grpc node's ack would have gone out on the DPS deployment-acks topic and never reached the sender at all.
This commit is contained in:
Joseph Doherty
+354
@@ -0,0 +1,354 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Cluster;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
||||
using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Runtime.Tests.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// The node-side gRPC ack sender (ClusterClient→gRPC migration Task 1B.2). The client is a fake,
|
||||
/// so what is under test is the sender's own logic: the ordered single-pass over
|
||||
/// <c>CentralGrpcEndpoints</c>, the per-call credentials, and the drop that used to be silent.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The load-bearing fact is that ONE endpoint is dialled on success.</b> This replaces an
|
||||
/// Akka <c>ClusterClient.Send</c>, which is single-target at-most-once — not a
|
||||
/// <c>SendToAll</c>. Both central nodes route an ack to the same
|
||||
/// <c>ConfigPublishCoordinator</c> singleton proxy, so posting to both would have the
|
||||
/// coordinator count this node twice and could seal a deployment on a doubled quorum.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The failure side matters just as much: the receptionist dropped an undeliverable ack in
|
||||
/// silence, and the whole point of the migration is that the same drop is now a Warning naming
|
||||
/// every endpoint and the status it answered with.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GrpcApplyAckSenderTests
|
||||
{
|
||||
private const string Psk = "psk-that-must-never-be-logged";
|
||||
|
||||
private static ApplyAck Ack() => new(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("site-a-1:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
private static GrpcApplyAckSender Sender(
|
||||
string[] endpoints,
|
||||
Func<string, CentralCommandService.CentralCommandServiceClient> factory,
|
||||
ILogger<GrpcApplyAckSender>? log = null,
|
||||
string[]? roles = null)
|
||||
=> new(
|
||||
Options.Create(new MeshTransportOptions
|
||||
{
|
||||
Mode = MeshTransportOptions.ModeGrpc,
|
||||
CentralGrpcEndpoints = endpoints,
|
||||
GrpcPsk = Psk,
|
||||
AckTimeoutSeconds = 7,
|
||||
}),
|
||||
Options.Create(new AkkaClusterOptions { Roles = roles ?? ["driver", "cluster-SITE-A"] }),
|
||||
log ?? NullLogger<GrpcApplyAckSender>.Instance,
|
||||
factory);
|
||||
|
||||
[Fact]
|
||||
public async Task A_successful_first_endpoint_is_the_ONLY_one_dialled()
|
||||
{
|
||||
// The single-count semantic. Two endpoints are configured for availability, not for fan-out:
|
||||
// an ack posted to both would reach the one coordinator singleton twice.
|
||||
var consulted = new List<string>();
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
var factory = (string endpoint) =>
|
||||
{
|
||||
consulted.Add(endpoint);
|
||||
return (CentralCommandService.CentralCommandServiceClient)new RecordingClient(calls);
|
||||
};
|
||||
|
||||
await Sender(["http://central-1:4055", "http://central-2:4055"], factory).SendAsync(Ack());
|
||||
|
||||
consulted.ShouldBe(["http://central-1:4055"]);
|
||||
calls.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_wire_message_carries_the_whole_ack()
|
||||
{
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
var ack = new ApplyAck(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("site-a-2:4053"),
|
||||
ApplyAckOutcome.Failed,
|
||||
"disk full",
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
await Sender(["http://central-1:4055"], _ => new RecordingClient(calls)).SendAsync(ack);
|
||||
|
||||
var sent = calls.Single().Request;
|
||||
sent.DeploymentId.ShouldBe(ack.DeploymentId.ToString());
|
||||
sent.NodeId.ShouldBe("site-a-2:4053");
|
||||
sent.Outcome.ShouldBe(ApplyOutcome.Failed);
|
||||
sent.FailureReason.ShouldBe("disk full");
|
||||
sent.CorrelationId.ShouldBe(ack.CorrelationId.ToString());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Every_call_carries_the_bearer_the_cluster_header_and_a_deadline()
|
||||
{
|
||||
// Central's CentralCommandAuthInterceptor grades the bearer against the key of the cluster
|
||||
// named in x-otopcua-cluster, so an ack missing either header is PermissionDenied — and the
|
||||
// deadline is what stops a dead first endpoint from stalling the ack forever.
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
var before = DateTime.UtcNow;
|
||||
|
||||
await Sender(["http://central-1:4055"], _ => new RecordingClient(calls)).SendAsync(Ack());
|
||||
|
||||
var options = calls.Single().Options;
|
||||
options.Headers.ShouldNotBeNull();
|
||||
options.Headers.GetValue("authorization").ShouldBe($"Bearer {Psk}");
|
||||
options.Headers.GetValue("x-otopcua-cluster").ShouldBe("SITE-A");
|
||||
options.Deadline.ShouldNotBeNull();
|
||||
options.Deadline!.Value.ShouldBeGreaterThan(before);
|
||||
options.Deadline!.Value.ShouldBeLessThanOrEqualTo(before.AddSeconds(7).AddSeconds(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unavailable_first_endpoint_fails_over_to_the_second()
|
||||
{
|
||||
var consulted = new List<string>();
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
var factory = (string endpoint) =>
|
||||
{
|
||||
consulted.Add(endpoint);
|
||||
return endpoint.Contains("central-1", StringComparison.Ordinal)
|
||||
? (CentralCommandService.CentralCommandServiceClient)
|
||||
new ThrowingClient(new RpcException(new Status(StatusCode.Unavailable, "down")))
|
||||
: new RecordingClient(calls);
|
||||
};
|
||||
|
||||
await Sender(["http://central-1:4055", "http://central-2:4055"], factory).SendAsync(Ack());
|
||||
|
||||
consulted.ShouldBe(["http://central-1:4055", "http://central-2:4055"]);
|
||||
calls.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task When_every_endpoint_fails_the_ack_is_dropped_with_one_naming_Warning()
|
||||
{
|
||||
// The pre-migration behaviour was a silent drop. It is still a drop — there is no queue and no
|
||||
// retry — but the operator now gets the node, the deployment, the correlation id and what each
|
||||
// endpoint actually answered, which is everything needed to tell "central is down" from
|
||||
// "central rejected this node's key".
|
||||
var log = new CapturingLogger<GrpcApplyAckSender>();
|
||||
var attempts = 0;
|
||||
var factory = (string endpoint) =>
|
||||
{
|
||||
attempts++;
|
||||
return (CentralCommandService.CentralCommandServiceClient)new ThrowingClient(
|
||||
new RpcException(new Status(
|
||||
endpoint.Contains("central-1", StringComparison.Ordinal)
|
||||
? StatusCode.Unavailable
|
||||
: StatusCode.PermissionDenied,
|
||||
"no")));
|
||||
};
|
||||
var ack = Ack();
|
||||
|
||||
await Sender(["http://central-1:4055", "http://central-2:4055"], factory, log).SendAsync(ack);
|
||||
|
||||
attempts.ShouldBe(2);
|
||||
var warning = log.Entries.Single(e => e.Level == LogLevel.Warning);
|
||||
warning.Message.ShouldContain(ack.NodeId.Value!);
|
||||
warning.Message.ShouldContain(ack.DeploymentId.ToString());
|
||||
warning.Message.ShouldContain(ack.CorrelationId.ToString());
|
||||
warning.Message.ShouldContain("Unavailable");
|
||||
warning.Message.ShouldContain("PermissionDenied");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_pass_never_posts_the_ack_twice()
|
||||
{
|
||||
// The counterpart of the first fact: failover walks forward only. A retry of an
|
||||
// already-delivered endpoint could double-count this node at the coordinator, which is the one
|
||||
// outcome worse than the ack being lost.
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
var consulted = new List<string>();
|
||||
var factory = (string endpoint) =>
|
||||
{
|
||||
consulted.Add(endpoint);
|
||||
return endpoint.Contains("c1", StringComparison.Ordinal)
|
||||
? (CentralCommandService.CentralCommandServiceClient)
|
||||
new ThrowingClient(new RpcException(new Status(StatusCode.Unavailable, "down")))
|
||||
: new RecordingClient(calls);
|
||||
};
|
||||
|
||||
await Sender(["http://c1:4055", "http://c2:4055", "http://c3:4055"], factory).SendAsync(Ack());
|
||||
|
||||
// c3 is never dialled: the pass stops at the first success, and it does not go back to c1.
|
||||
consulted.ShouldBe(["http://c1:4055", "http://c2:4055"]);
|
||||
calls.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_pre_shared_key_is_never_written_to_a_log()
|
||||
{
|
||||
// The bearer travels in the call headers on every attempt; the failure log names endpoints and
|
||||
// status codes only. A per-endpoint dump of the Metadata would put the cluster's PSK in the
|
||||
// node log of every deployment that could not reach central.
|
||||
var log = new CapturingLogger<GrpcApplyAckSender>();
|
||||
var factory = (string _) => (CentralCommandService.CentralCommandServiceClient)
|
||||
new ThrowingClient(new RpcException(new Status(StatusCode.Unavailable, "down")));
|
||||
|
||||
await Sender(["http://central-1:4055"], factory, log).SendAsync(Ack());
|
||||
|
||||
log.Entries.ShouldNotBeEmpty();
|
||||
log.Entries.Any(e => e.Message.Contains(Psk, StringComparison.Ordinal)).ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_empty_endpoint_list_is_a_logged_drop_not_a_throw()
|
||||
{
|
||||
// The validator refuses to start a Grpc driver node with no endpoints, so this is reachable
|
||||
// only through a code path that built the sender for a node the validator never graded (a test
|
||||
// harness, an admin-only graph). It must still be a visible drop rather than an exception on a
|
||||
// task the actor discards, which would vanish entirely.
|
||||
var log = new CapturingLogger<GrpcApplyAckSender>();
|
||||
var dialled = false;
|
||||
var factory = (string _) =>
|
||||
{
|
||||
dialled = true;
|
||||
return (CentralCommandService.CentralCommandServiceClient)new RecordingClient([]);
|
||||
};
|
||||
|
||||
await Sender([], factory, log).SendAsync(Ack());
|
||||
|
||||
dialled.ShouldBeFalse();
|
||||
log.Entries.Single(e => e.Level == LogLevel.Warning).Message
|
||||
.ShouldContain("CentralGrpcEndpoints");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_node_with_no_cluster_role_still_posts_and_is_warned_about()
|
||||
{
|
||||
// No cluster-scoped role means no x-otopcua-cluster header, and central has no key to grade the
|
||||
// bearer against — every ack will come back PermissionDenied. Posting anyway (rather than
|
||||
// refusing locally) keeps the rejection where it can be read on both sides, and the local
|
||||
// Warning explains a PermissionDenied that would otherwise look like a key mismatch.
|
||||
var log = new CapturingLogger<GrpcApplyAckSender>();
|
||||
var calls = new List<(ApplyAckMessage Request, CallOptions Options)>();
|
||||
|
||||
await Sender(["http://central-1:4055"], _ => new RecordingClient(calls), log, roles: ["driver"])
|
||||
.SendAsync(Ack());
|
||||
|
||||
calls.Single().Options.Headers!.Any(h =>
|
||||
h.Key.Equals("x-otopcua-cluster", StringComparison.Ordinal)).ShouldBeFalse();
|
||||
log.Entries.Any(e => e.Level == LogLevel.Warning
|
||||
&& e.Message.Contains("cluster-", StringComparison.Ordinal)).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_unexpected_client_fault_is_a_logged_drop_not_an_unobserved_task_fault()
|
||||
{
|
||||
// The caller is an actor that discards the returned Task, so anything escaping here would be
|
||||
// swallowed by the runtime and rediscovered as an unobserved exception at GC — the exact
|
||||
// silent drop this transport exists to remove.
|
||||
var log = new CapturingLogger<GrpcApplyAckSender>();
|
||||
var factory = (string _) => (CentralCommandService.CentralCommandServiceClient)
|
||||
new FaultingClient(new InvalidOperationException("channel disposed"));
|
||||
|
||||
await Should.NotThrowAsync(() => Sender(["http://central-1:4055"], factory, log).SendAsync(Ack()));
|
||||
|
||||
log.Entries.Any(e => e.Level == LogLevel.Warning).ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Dispose_is_idempotent()
|
||||
{
|
||||
var sender = Sender(["http://central-1:4055"], _ => new RecordingClient([]));
|
||||
|
||||
sender.Dispose();
|
||||
Should.NotThrow(sender.Dispose);
|
||||
}
|
||||
|
||||
// ---- fakes --------------------------------------------------------------------------------
|
||||
|
||||
private static AsyncUnaryCall<ReportAck> Ok() => new(
|
||||
Task.FromResult(new ReportAck()),
|
||||
Task.FromResult(new Metadata()),
|
||||
() => Status.DefaultSuccess,
|
||||
() => [],
|
||||
() => { });
|
||||
|
||||
/// <summary>A generated client double recording the request and the call options it was given.</summary>
|
||||
private sealed class RecordingClient(List<(ApplyAckMessage Request, CallOptions Options)> calls)
|
||||
: CentralCommandService.CentralCommandServiceClient
|
||||
{
|
||||
public override AsyncUnaryCall<ReportAck> ReportApplyAsync(ApplyAckMessage request, CallOptions options)
|
||||
{
|
||||
calls.Add((request, options));
|
||||
return Ok();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A generated client double whose call throws — an endpoint that is down or refuses.</summary>
|
||||
private sealed class ThrowingClient(RpcException toThrow) : CentralCommandService.CentralCommandServiceClient
|
||||
{
|
||||
public override AsyncUnaryCall<ReportAck> ReportApplyAsync(ApplyAckMessage request, CallOptions options)
|
||||
=> throw toThrow;
|
||||
}
|
||||
|
||||
/// <summary>A generated client double throwing something that is NOT an <see cref="RpcException"/>.</summary>
|
||||
private sealed class FaultingClient(Exception toThrow) : CentralCommandService.CentralCommandServiceClient
|
||||
{
|
||||
public override AsyncUnaryCall<ReportAck> ReportApplyAsync(ApplyAckMessage request, CallOptions options)
|
||||
=> throw toThrow;
|
||||
}
|
||||
|
||||
/// <summary>Test logger that captures all log entries.</summary>
|
||||
/// <typeparam name="T">The category type.</typeparam>
|
||||
private sealed class CapturingLogger<T> : ILogger<T>
|
||||
{
|
||||
/// <summary>Gets the captured log entries.</summary>
|
||||
public List<(LogLevel Level, string Message, Exception? Exception)> Entries { get; } = [];
|
||||
|
||||
/// <summary>Begins a scope (stub implementation).</summary>
|
||||
/// <typeparam name="TState">The type of the scope state.</typeparam>
|
||||
/// <param name="state">The scope state.</param>
|
||||
/// <returns>A no-op scope.</returns>
|
||||
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
|
||||
|
||||
/// <summary>Checks if logging is enabled (always true).</summary>
|
||||
/// <param name="logLevel">The log level to check.</param>
|
||||
/// <returns><c>true</c>.</returns>
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
/// <summary>Logs an entry and captures it.</summary>
|
||||
/// <typeparam name="TState">The type of the log state.</typeparam>
|
||||
/// <param name="logLevel">The log level.</param>
|
||||
/// <param name="eventId">The event ID.</param>
|
||||
/// <param name="state">The log state.</param>
|
||||
/// <param name="exception">The exception, if any.</param>
|
||||
/// <param name="formatter">The log message formatter.</param>
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel, EventId eventId, TState state, Exception? exception,
|
||||
Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
Entries.Add((logLevel, formatter(state, exception), exception));
|
||||
}
|
||||
|
||||
private sealed class NullScope : IDisposable
|
||||
{
|
||||
/// <summary>Gets the singleton instance.</summary>
|
||||
public static NullScope Instance { get; } = new();
|
||||
|
||||
/// <summary>No-op.</summary>
|
||||
public void Dispose() { }
|
||||
}
|
||||
}
|
||||
}
|
||||
+99
@@ -127,4 +127,103 @@ public class NodeCommunicationActorTests : RuntimeActorTestBase
|
||||
EventFilter.Warning(contains: "dropping ApplyAck").ExpectOne(
|
||||
TimeSpan.FromSeconds(5), () => actor.Tell(ack));
|
||||
}
|
||||
|
||||
// ---- Mode=Grpc (ClusterClient→gRPC migration Task 1B.2) -------------------------------------
|
||||
|
||||
private static ApplyAck NewAck() => new(
|
||||
new DeploymentId(Guid.NewGuid()),
|
||||
new NodeId("host-a:4053"),
|
||||
ApplyAckOutcome.Applied,
|
||||
null,
|
||||
new CorrelationId(Guid.NewGuid()));
|
||||
|
||||
[Fact]
|
||||
public void Under_Grpc_the_ack_goes_to_the_sender_exactly_once()
|
||||
{
|
||||
// The gRPC counterpart of the ClusterClient.Send fact above, and it carries the same
|
||||
// single-count obligation: one post per node per dispatch, because both central endpoints
|
||||
// route to the SAME coordinator singleton proxy. Where the ack goes among the endpoints is the
|
||||
// sender's business (GrpcApplyAckSenderTests); what this pins is that the actor hands it over
|
||||
// once and does not also fan it out itself.
|
||||
var sender = new RecordingAckSender();
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null, ackSender: sender));
|
||||
|
||||
var ack = NewAck();
|
||||
actor.Tell(ack);
|
||||
|
||||
AwaitAssert(() => sender.Sent.Count.ShouldBe(1), TimeSpan.FromSeconds(5));
|
||||
sender.Sent[0].ShouldBe(ack);
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
sender.Sent.Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_sender_wins_over_a_cluster_client_and_the_ack_is_not_sent_twice()
|
||||
{
|
||||
// Defence in depth. The registration arm builds exactly one transport — under Grpc no
|
||||
// ClusterClient is created at all — but if both were ever present, posting over both would
|
||||
// deliver two acks for one apply and the coordinator would count this node twice.
|
||||
var client = CreateTestProbe();
|
||||
var sender = new RecordingAckSender();
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(client.Ref, sender));
|
||||
|
||||
actor.Tell(NewAck());
|
||||
|
||||
AwaitAssert(() => sender.Sent.Count.ShouldBe(1), TimeSpan.FromSeconds(5));
|
||||
client.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_ClusterClient_branch_never_touches_the_sender()
|
||||
{
|
||||
// Null-safe by construction: under Dps and ClusterClient the sender is not built at all, so the
|
||||
// ApplyAck path must not depend on one existing.
|
||||
var client = CreateTestProbe();
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(client.Ref));
|
||||
|
||||
var ack = NewAck();
|
||||
actor.Tell(ack);
|
||||
|
||||
client.ExpectMsg<ClusterClient.Send>(TimeSpan.FromSeconds(5)).Message.ShouldBe(ack);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Inbound_commands_are_unaffected_by_the_presence_of_a_sender()
|
||||
{
|
||||
// The sender is outbound-only. A Grpc node still re-emits central's commands on its own
|
||||
// EventStream exactly as every other mode does.
|
||||
var subscriber = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(subscriber.Ref, typeof(DispatchDeployment));
|
||||
var sender = new RecordingAckSender();
|
||||
var actor = Sys.ActorOf(NodeCommunicationActor.Props(centralClient: null, ackSender: sender));
|
||||
|
||||
var dispatch = NewDispatch();
|
||||
actor.Tell(dispatch);
|
||||
|
||||
subscriber.ExpectMsg<DispatchDeployment>(TimeSpan.FromSeconds(5)).ShouldBe(dispatch);
|
||||
sender.Sent.ShouldBeEmpty();
|
||||
}
|
||||
|
||||
/// <summary>An <see cref="IApplyAckSender"/> double recording what the actor handed it.</summary>
|
||||
private sealed class RecordingAckSender : IApplyAckSender
|
||||
{
|
||||
private readonly Lock _gate = new();
|
||||
private readonly List<ApplyAck> _sent = [];
|
||||
|
||||
/// <summary>Gets a snapshot of the acks handed to this sender.</summary>
|
||||
public IReadOnlyList<ApplyAck> Sent
|
||||
{
|
||||
get { lock (_gate) return [.. _sent]; }
|
||||
}
|
||||
|
||||
/// <summary>Records <paramref name="ack"/>.</summary>
|
||||
/// <param name="ack">The ack the actor handed over.</param>
|
||||
/// <param name="cancellationToken">Unused.</param>
|
||||
/// <returns>A completed task — the real sender never throws either.</returns>
|
||||
public Task SendAsync(ApplyAck ack, CancellationToken cancellationToken = default)
|
||||
{
|
||||
lock (_gate) _sent.Add(ack);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user