2c45c3238b
Additive RequestedBy on Retry/DiscardSiteCallRequest, plumbed from the Central UI. On an Applied relay, SiteCallAuditActor emits one best-effort CachedResolve central direct-write row (Submitted/Discarded) with the operator as Actor. Gated on an injected ICentralAuditWriter (null in existing tests → no mirror read, unchanged); the site remains the source of truth for the state change.
372 lines
16 KiB
C#
372 lines
16 KiB
C#
using Akka.Actor;
|
|
using Akka.TestKit.Xunit2;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Communication;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests;
|
|
|
|
/// <summary>
|
|
/// Task 5 (#22 Retry/Discard relay): tests for <see cref="SiteCallAuditActor"/>
|
|
/// relaying operator Retry/Discard on a parked Site Call down to the owning
|
|
/// site. The relay routes a <see cref="RetryParkedOperation"/> /
|
|
/// <see cref="DiscardParkedOperation"/> command via a <see cref="SiteEnvelope"/>
|
|
/// to the <see cref="ZB.MOM.WW.ScadaBridge.Communication.Actors.CentralCommunicationActor"/>
|
|
/// (stood in by a <c>TestProbe</c> here) and awaits the site's
|
|
/// <see cref="ParkedOperationActionAck"/>. These tests never touch the
|
|
/// <c>SiteCalls</c> repository — central never mutates the mirror row.
|
|
/// </summary>
|
|
public class SiteCallRelayTests : TestKit
|
|
{
|
|
/// <summary>
|
|
/// A repository that fails every call — the relay path must NEVER touch the
|
|
/// <c>SiteCalls</c> table (central is not the source of truth), so any
|
|
/// invocation here is a test failure surfaced as an exception.
|
|
/// </summary>
|
|
private sealed class ThrowingRepository : ISiteCallAuditRepository
|
|
{
|
|
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not write the SiteCalls row");
|
|
|
|
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not read the SiteCalls row");
|
|
|
|
public Task<IReadOnlyList<SiteCall>> QueryAsync(
|
|
SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not query the SiteCalls table");
|
|
|
|
public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not purge");
|
|
|
|
public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute KPIs");
|
|
|
|
public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute per-site KPIs");
|
|
|
|
public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute per-node KPIs");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Builds a <see cref="SiteCallAuditActor"/> with a throwing repository and a
|
|
/// short relay timeout, and registers <paramref name="centralComm"/> as the
|
|
/// central→site transport.
|
|
/// </summary>
|
|
private IActorRef CreateActor(IActorRef centralComm)
|
|
{
|
|
var options = new SiteCallAuditOptions { RelayTimeout = TimeSpan.FromMilliseconds(500) };
|
|
var actor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
|
new ThrowingRepository(),
|
|
NullLogger<SiteCallAuditActor>.Instance,
|
|
options)));
|
|
actor.Tell(new RegisterCentralCommunication(centralComm));
|
|
return actor;
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_RoutesRetryParkedOperation_ToOwningSite()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
var id = Guid.NewGuid();
|
|
actor.Tell(new RetrySiteCallRequest("corr-1", id, "site-north"));
|
|
|
|
// The relay must wrap a RetryParkedOperation in a SiteEnvelope addressed
|
|
// to the owning site.
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
Assert.Equal("site-north", envelope.SiteId);
|
|
var relay = Assert.IsType<RetryParkedOperation>(envelope.Message);
|
|
Assert.Equal(id, relay.TrackedOperationId.Value);
|
|
|
|
// The site applies it and acks; the relay reports Applied.
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: true));
|
|
|
|
var response = ExpectMsg<RetrySiteCallResponse>();
|
|
Assert.Equal("corr-1", response.CorrelationId);
|
|
Assert.Equal(SiteCallRelayOutcome.Applied, response.Outcome);
|
|
Assert.True(response.Success);
|
|
Assert.True(response.SiteReachable);
|
|
Assert.Null(response.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DiscardSiteCall_RoutesDiscardParkedOperation_ToOwningSite()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
var id = Guid.NewGuid();
|
|
actor.Tell(new DiscardSiteCallRequest("corr-2", id, "site-south"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
Assert.Equal("site-south", envelope.SiteId);
|
|
var relay = Assert.IsType<DiscardParkedOperation>(envelope.Message);
|
|
Assert.Equal(id, relay.TrackedOperationId.Value);
|
|
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: true));
|
|
|
|
var response = ExpectMsg<DiscardSiteCallResponse>();
|
|
Assert.Equal(SiteCallRelayOutcome.Applied, response.Outcome);
|
|
Assert.True(response.Success);
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_SiteRepliesNotApplied_ReportsNotParked()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-3", Guid.NewGuid(), "site-north"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
var relay = (RetryParkedOperation)envelope.Message;
|
|
// The site found nothing parked — a definitive answer, not a failure.
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: false));
|
|
|
|
var response = ExpectMsg<RetrySiteCallResponse>();
|
|
Assert.Equal(SiteCallRelayOutcome.NotParked, response.Outcome);
|
|
Assert.False(response.Success);
|
|
Assert.True(response.SiteReachable);
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_SiteRepliesError_ReportsOperationFailed()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-4", Guid.NewGuid(), "site-north"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
var relay = (RetryParkedOperation)envelope.Message;
|
|
central.Reply(new ParkedOperationActionAck(
|
|
relay.CorrelationId, Applied: false, "Parked message handler not available"));
|
|
|
|
var response = ExpectMsg<RetrySiteCallResponse>();
|
|
Assert.Equal(SiteCallRelayOutcome.OperationFailed, response.Outcome);
|
|
Assert.False(response.Success);
|
|
// The site WAS reached — this is an operation failure, not unreachable.
|
|
Assert.True(response.SiteReachable);
|
|
Assert.NotNull(response.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_SiteNeverReplies_ReportsSiteUnreachable()
|
|
{
|
|
// A central comm probe that silently drops the relay — models an offline
|
|
// site / no ClusterClient route: the Ask times out.
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-5", Guid.NewGuid(), "site-offline"));
|
|
|
|
central.ExpectMsg<SiteEnvelope>();
|
|
// Probe does not reply — the relay Ask times out (RelayTimeout = 500ms).
|
|
|
|
var response = ExpectMsg<RetrySiteCallResponse>(TimeSpan.FromSeconds(3));
|
|
Assert.Equal(SiteCallRelayOutcome.SiteUnreachable, response.Outcome);
|
|
Assert.False(response.Success);
|
|
// The distinct unreachable signal the UI relies on.
|
|
Assert.False(response.SiteReachable);
|
|
Assert.NotNull(response.ErrorMessage);
|
|
}
|
|
|
|
[Fact]
|
|
public void DiscardSiteCall_SiteNeverReplies_ReportsSiteUnreachable()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var actor = CreateActor(central.Ref);
|
|
|
|
actor.Tell(new DiscardSiteCallRequest("corr-6", Guid.NewGuid(), "site-offline"));
|
|
|
|
central.ExpectMsg<SiteEnvelope>();
|
|
|
|
var response = ExpectMsg<DiscardSiteCallResponse>(TimeSpan.FromSeconds(3));
|
|
Assert.Equal(SiteCallRelayOutcome.SiteUnreachable, response.Outcome);
|
|
Assert.False(response.SiteReachable);
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_BeforeCentralCommunicationRegistered_ReportsSiteUnreachable()
|
|
{
|
|
// No RegisterCentralCommunication — the actor has no transport to reach
|
|
// any site, so the only honest answer is "unreachable".
|
|
var options = new SiteCallAuditOptions { RelayTimeout = TimeSpan.FromMilliseconds(500) };
|
|
var actor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
|
new ThrowingRepository(),
|
|
NullLogger<SiteCallAuditActor>.Instance,
|
|
options)));
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-7", Guid.NewGuid(), "site-north"));
|
|
|
|
var response = ExpectMsg<RetrySiteCallResponse>();
|
|
Assert.Equal(SiteCallRelayOutcome.SiteUnreachable, response.Outcome);
|
|
Assert.False(response.SiteReachable);
|
|
}
|
|
|
|
// ── Task 14: operator-identity audit on a successful relay ──
|
|
|
|
/// <summary>Captures the central direct-write audit events emitted by the relay.</summary>
|
|
private sealed class RecordingCentralAuditWriter : ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ICentralAuditWriter
|
|
{
|
|
public List<AuditRowProjection.AuditRowValues> Events { get; } = new();
|
|
|
|
public Task WriteAsync(ZB.MOM.WW.Audit.AuditEvent evt, CancellationToken ct = default)
|
|
{
|
|
lock (Events)
|
|
{
|
|
Events.Add(evt.AsRow());
|
|
}
|
|
|
|
return Task.CompletedTask;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A repository whose <see cref="GetAsync"/> returns one fixed row (so the relay
|
|
/// can read its Channel for the audit event) but still throws on every MUTATING
|
|
/// call — central never writes the mirror row on the relay path.
|
|
/// </summary>
|
|
private sealed class RowReturningRepository : ISiteCallAuditRepository
|
|
{
|
|
private readonly SiteCall _row;
|
|
public RowReturningRepository(SiteCall row) { _row = row; }
|
|
|
|
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
|
|
Task.FromResult<SiteCall?>(_row.TrackedOperationId == id ? _row : null);
|
|
|
|
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not write the SiteCalls row");
|
|
|
|
public Task<IReadOnlyList<SiteCall>> QueryAsync(
|
|
SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not query the SiteCalls table");
|
|
|
|
public Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not purge");
|
|
|
|
public Task<SiteCallKpiSnapshot> ComputeKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute KPIs");
|
|
|
|
public Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute per-site KPIs");
|
|
|
|
public Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
|
|
DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default) =>
|
|
throw new InvalidOperationException("relay must not compute per-node KPIs");
|
|
}
|
|
|
|
private static SiteCall MakeRow(Guid id, string channel = "ApiOutbound") => new()
|
|
{
|
|
TrackedOperationId = new TrackedOperationId(id),
|
|
Channel = channel,
|
|
Target = "ERP.GetOrder",
|
|
SourceSite = "site-north",
|
|
SourceNode = "node-a",
|
|
Status = "Parked",
|
|
RetryCount = 3,
|
|
CreatedAtUtc = DateTime.UtcNow,
|
|
UpdatedAtUtc = DateTime.UtcNow,
|
|
IngestedAtUtc = DateTime.UtcNow,
|
|
};
|
|
|
|
private (IActorRef Actor, RecordingCentralAuditWriter Writer) CreateActorWithAudit(IActorRef centralComm, SiteCall row)
|
|
{
|
|
var writer = new RecordingCentralAuditWriter();
|
|
var options = new SiteCallAuditOptions { RelayTimeout = TimeSpan.FromMilliseconds(500) };
|
|
var actor = Sys.ActorOf(Props.Create(() => new SiteCallAuditActor(
|
|
new RowReturningRepository(row),
|
|
NullLogger<SiteCallAuditActor>.Instance,
|
|
options,
|
|
(ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ICentralAuditWriter)writer)));
|
|
actor.Tell(new RegisterCentralCommunication(centralComm));
|
|
return (actor, writer);
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_WhenApplied_EmitsCachedResolveSubmitted_WithOperator()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var id = Guid.NewGuid();
|
|
var (actor, writer) = CreateActorWithAudit(central.Ref, MakeRow(id, channel: "ApiOutbound"));
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-a", id, "site-north", RequestedBy: "jdoe"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
var relay = (RetryParkedOperation)envelope.Message;
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: true));
|
|
|
|
ExpectMsg<RetrySiteCallResponse>(r => r.Success);
|
|
AwaitAssert(() =>
|
|
{
|
|
List<AuditRowProjection.AuditRowValues> events;
|
|
lock (writer.Events) { events = writer.Events.ToList(); }
|
|
Assert.Single(events);
|
|
var evt = events[0];
|
|
Assert.Equal(Commons.Types.Enums.AuditChannel.ApiOutbound, evt.Channel);
|
|
Assert.Equal(Commons.Types.Enums.AuditKind.CachedResolve, evt.Kind);
|
|
Assert.Equal(Commons.Types.Enums.AuditStatus.Submitted, evt.Status);
|
|
Assert.Equal("jdoe", evt.Actor);
|
|
Assert.Equal(id, evt.CorrelationId);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void RetrySiteCall_WhenNotApplied_EmitsNoAuditRow()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var id = Guid.NewGuid();
|
|
var (actor, writer) = CreateActorWithAudit(central.Ref, MakeRow(id));
|
|
|
|
actor.Tell(new RetrySiteCallRequest("corr-b", id, "site-north", RequestedBy: "jdoe"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
var relay = (RetryParkedOperation)envelope.Message;
|
|
// Nothing was parked at the site — no state change, so no operator audit row.
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: false));
|
|
|
|
ExpectMsg<RetrySiteCallResponse>(r => r.Outcome == SiteCallRelayOutcome.NotParked);
|
|
// Give any erroneous emission a chance to land, then assert none did.
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
lock (writer.Events) { Assert.Empty(writer.Events); }
|
|
}
|
|
|
|
[Fact]
|
|
public void DiscardSiteCall_WhenApplied_EmitsCachedResolveDiscarded_WithOperator()
|
|
{
|
|
var central = CreateTestProbe();
|
|
var id = Guid.NewGuid();
|
|
var (actor, writer) = CreateActorWithAudit(central.Ref, MakeRow(id, channel: "DbOutbound"));
|
|
|
|
actor.Tell(new DiscardSiteCallRequest("corr-c", id, "site-north", RequestedBy: "jdoe"));
|
|
|
|
var envelope = central.ExpectMsg<SiteEnvelope>();
|
|
var relay = (DiscardParkedOperation)envelope.Message;
|
|
central.Reply(new ParkedOperationActionAck(relay.CorrelationId, Applied: true));
|
|
|
|
ExpectMsg<DiscardSiteCallResponse>(r => r.Success);
|
|
AwaitAssert(() =>
|
|
{
|
|
List<AuditRowProjection.AuditRowValues> events;
|
|
lock (writer.Events) { events = writer.Events.ToList(); }
|
|
Assert.Single(events);
|
|
var evt = events[0];
|
|
Assert.Equal(Commons.Types.Enums.AuditChannel.DbOutbound, evt.Channel);
|
|
Assert.Equal(Commons.Types.Enums.AuditStatus.Discarded, evt.Status);
|
|
Assert.Equal("jdoe", evt.Actor);
|
|
});
|
|
}
|
|
}
|