perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull
This commit is contained in:
+8
-8
@@ -117,7 +117,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
// Endpoint resolution + request shaping.
|
||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||
@@ -141,7 +141,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -161,7 +161,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
// MUST NOT throw — per the IPullAuditEventsClient contract.
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -178,7 +178,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -197,7 +197,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -221,7 +221,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
// MUST NOT throw — must dial successfully.
|
||||
var result = await sut.PullAsync("site-a", minUnspecified, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", minUnspecified, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, invoker.CallCount);
|
||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||
@@ -252,7 +252,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
|
||||
var evt = Assert.Single(result.Events);
|
||||
@@ -271,7 +271,7 @@ public class GrpcPullAuditEventsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullAuditEventsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.Events);
|
||||
Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
|
||||
|
||||
+101
-3
@@ -194,7 +194,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
/// </summary>
|
||||
private sealed class ScriptedPullClient : IPullAuditEventsClient
|
||||
{
|
||||
public List<(string SiteId, DateTime SinceUtc, int BatchSize)> Calls { get; } = new();
|
||||
public List<(string SiteId, DateTime SinceUtc, string? AfterId, int BatchSize)> Calls { get; } = new();
|
||||
private readonly Dictionary<string, Queue<PullAuditEventsResponse>> _scripted = new();
|
||||
private readonly Dictionary<string, Exception> _throwOnSite = new();
|
||||
|
||||
@@ -211,9 +211,9 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
}
|
||||
|
||||
public Task<PullAuditEventsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
Calls.Add((siteId, sinceUtc, batchSize));
|
||||
Calls.Add((siteId, sinceUtc, afterId, batchSize));
|
||||
if (_throwOnSite.TryGetValue(siteId, out var ex))
|
||||
{
|
||||
throw ex;
|
||||
@@ -425,6 +425,104 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
|
||||
Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
|
||||
Assert.Equal(t3, client.Calls[1].SinceUtc);
|
||||
|
||||
// The composite half travels too (arch-review phase-2 residual #5): the first pull has
|
||||
// no cursor at all, the second carries the id of the row at t3 so the site can retire
|
||||
// it exactly instead of leaving the boundary instant permanently servable.
|
||||
Assert.Null(client.Calls[0].AfterId);
|
||||
Assert.Equal(e3.EventId.ToString(), client.Calls[1].AfterId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Cursor_AdvancesOnTheIdTiebreak_WhenEveryRowSharesOneInstant()
|
||||
{
|
||||
// The case a bare timestamp cursor could never drain: a burst all stamped at the same
|
||||
// instant. The timestamp cannot move, so only the id half can — and it must, or the
|
||||
// next pull re-serves the identical window forever.
|
||||
var sites = new StaticEnumerator(new SiteEntry("siteA", "http://siteA:8083"));
|
||||
var t = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// Deterministic ids so "the greatest ordinal" is a fact, not a coin flip.
|
||||
var low = NewEvent("siteA", t, Guid.Parse("11111111-1111-1111-1111-111111111111"));
|
||||
var high = NewEvent("siteA", t, Guid.Parse("99999999-9999-9999-9999-999999999999"));
|
||||
var mid = NewEvent("siteA", t, Guid.Parse("55555555-5555-5555-5555-555555555555"));
|
||||
|
||||
var client = new ScriptedPullClient().Script("siteA",
|
||||
new PullAuditEventsResponse(new[] { low, high, mid }, MoreAvailable: true));
|
||||
var repo = new RecordingRepo();
|
||||
|
||||
CreateActor(sites, client, repo, FastTickOptions());
|
||||
|
||||
AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
|
||||
$"need at least 2 pulls, got {client.Calls.Count}"),
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
Assert.Equal(t, client.Calls[1].SinceUtc);
|
||||
Assert.Equal(high.EventId.ToString(), client.Calls[1].AfterId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BothCursorHalves_AreHeldBack_WhileARowIsStillBeingRetried()
|
||||
{
|
||||
// A held-back cursor must hold BOTH halves: advancing the id while pinning the
|
||||
// timestamp would skip past the very rows being retried.
|
||||
var sites = new StaticEnumerator(new SiteEntry("siteA", "http://siteA:8083"));
|
||||
var t = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
var evt = NewEvent("siteA", t);
|
||||
|
||||
var client = new ScriptedPullClient().Script("siteA",
|
||||
new PullAuditEventsResponse(new[] { evt }, MoreAvailable: false));
|
||||
var repo = new AlwaysThrowingRepo();
|
||||
|
||||
CreateActor(sites, client, repo, FastTickOptions());
|
||||
|
||||
AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
|
||||
$"need at least 2 pulls, got {client.Calls.Count}"),
|
||||
duration: TimeSpan.FromSeconds(5),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
Assert.Equal(DateTime.MinValue, client.Calls[1].SinceUtc);
|
||||
Assert.Null(client.Calls[1].AfterId);
|
||||
}
|
||||
|
||||
/// <summary>Repository whose every insert throws, so the retry hold-back path is taken.</summary>
|
||||
private sealed class AlwaysThrowingRepo : IAuditLogRepository
|
||||
{
|
||||
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default) =>
|
||||
throw new InvalidOperationException("central insert failed");
|
||||
|
||||
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(
|
||||
DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null,
|
||||
CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
string sentinel, DateTime before, int batchSize, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
|
||||
DateTime threshold, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
|
||||
TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
|
||||
Guid executionId, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
@@ -85,7 +85,7 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
|
||||
}
|
||||
|
||||
public async Task<PullAuditEventsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
CallCount++;
|
||||
|
||||
@@ -94,16 +94,16 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
|
||||
// is retired FIRST; the rows this call serves are NOT retired, because
|
||||
// nothing yet proves central consumed them. A fault between here and
|
||||
// central's commit therefore re-serves them on the next tick instead of
|
||||
// losing them. The actor sends no after_id, so the cursor is a bare
|
||||
// timestamp under the inclusive >= read contract and only rows strictly
|
||||
// older than it are provably received.
|
||||
// losing them. The actor now sends the composite (timestamp, id) cursor,
|
||||
// so retirement is exact: the rows AT the cursor instant are proven
|
||||
// received too, where a bare timestamp could only prove strictly-older ones.
|
||||
if (sinceUtc > DateTime.MinValue)
|
||||
{
|
||||
await _siteQueue.MarkReconciledUpToAsync(sinceUtc, null, ct).ConfigureAwait(false);
|
||||
await _siteQueue.MarkReconciledUpToAsync(sinceUtc, afterId, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var rows = await _siteQueue
|
||||
.ReadPendingSinceAsync(sinceUtc, batchSize, afterId: null, ct)
|
||||
.ReadPendingSinceAsync(sinceUtc, batchSize, afterId, ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// MoreAvailable is true iff the read filled the batch — the actor
|
||||
|
||||
@@ -331,6 +331,14 @@ public class AlarmSummaryRenderTests : BunitContext
|
||||
|
||||
public bool IsLive(int siteId) => _live;
|
||||
|
||||
/// <summary>
|
||||
/// The aggregator's own fan-out result. Settable so a test can prove the page renders
|
||||
/// the not-reporting list sourced from the live cache while it is serving the site.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> NotReporting { get; set; } = Array.Empty<string>();
|
||||
|
||||
public IReadOnlyList<string> GetNotReportingInstances(int siteId) => NotReporting;
|
||||
|
||||
public void PushAlarms(IReadOnlyList<AlarmStateChanged> alarms)
|
||||
{
|
||||
_current = alarms;
|
||||
|
||||
@@ -105,6 +105,8 @@ public class AlarmSummaryVirtualizeTests : BunitContext
|
||||
|
||||
public bool IsLive(int siteId) => false;
|
||||
|
||||
public IReadOnlyList<string> GetNotReportingInstances(int siteId) => Array.Empty<string>();
|
||||
|
||||
private sealed class NoOp : IDisposable
|
||||
{
|
||||
public void Dispose() { }
|
||||
|
||||
+54
-19
@@ -56,13 +56,13 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
_provider = services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private SharedAlarmSummaryService CreateSut(TimeSpan liveCacheTtl) =>
|
||||
new(_provider.GetRequiredService<IServiceScopeFactory>(), _liveCache, liveCacheTtl, () => _now);
|
||||
private SharedAlarmSummaryService CreateSut() =>
|
||||
new(_provider.GetRequiredService<IServiceScopeFactory>(), _liveCache, () => _now);
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentCircuits_ShareOneFanOut()
|
||||
{
|
||||
var sut = CreateSut(TimeSpan.FromSeconds(60));
|
||||
var sut = CreateSut();
|
||||
|
||||
var results = await Task.WhenAll(Enumerable.Range(0, 8).Select(_ => sut.GetSiteAlarmsAsync(SiteId)));
|
||||
|
||||
@@ -73,7 +73,7 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
[Fact]
|
||||
public async Task ColdLiveCache_RefreshesWithinThePageTick()
|
||||
{
|
||||
var sut = CreateSut(TimeSpan.FromSeconds(60));
|
||||
var sut = CreateSut();
|
||||
_liveCache.Live = false;
|
||||
|
||||
await sut.GetSiteAlarmsAsync(SiteId);
|
||||
@@ -86,22 +86,51 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LiveCacheServing_WidensTheWindowToTheReconcileInterval()
|
||||
public async Task LiveCacheServing_SkipsTheFanOutEntirely()
|
||||
{
|
||||
var sut = CreateSut(TimeSpan.FromSeconds(60));
|
||||
// The aggregator's seed/reconcile already ran this exact fan-out and publishes both
|
||||
// halves of the answer, so the façade must not run a second one — not now, not after
|
||||
// any elapsed window (arch-review phase-2 residual #4).
|
||||
var sut = CreateSut();
|
||||
_liveCache.Live = true;
|
||||
_liveCache.NotReporting = new[] { "inst-silent" };
|
||||
_liveCache.Current = new[]
|
||||
{
|
||||
new AlarmStateChanged("inst-a", "A-alarm", AlarmState.Active, 500, T0),
|
||||
};
|
||||
|
||||
var first = await sut.GetSiteAlarmsAsync(SiteId);
|
||||
_now = T0.AddSeconds(61);
|
||||
var second = await sut.GetSiteAlarmsAsync(SiteId);
|
||||
|
||||
await _instanceRepo.DidNotReceive().GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>());
|
||||
await _snapshotClient.DidNotReceive().GetSnapshotAsync(
|
||||
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<CancellationToken>());
|
||||
|
||||
// Both halves come from the cache: the rows so a liveness flip mid-call lands on
|
||||
// last-known state instead of a blank grid, and the not-reporting names the page shows.
|
||||
foreach (var result in new[] { first, second })
|
||||
{
|
||||
Assert.Equal("A-alarm", Assert.Single(result.Alarms).Alarm.AlarmName);
|
||||
Assert.Equal("inst-silent", Assert.Single(result.NotReportingInstances));
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task LiveCacheGoingCold_FallsBackToTheFanOut()
|
||||
{
|
||||
var sut = CreateSut();
|
||||
_liveCache.Live = true;
|
||||
|
||||
await sut.GetSiteAlarmsAsync(SiteId);
|
||||
_now = T0.AddSeconds(30);
|
||||
await sut.GetSiteAlarmsAsync(SiteId);
|
||||
await _instanceRepo.DidNotReceive().GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>());
|
||||
|
||||
// Aggregator died / stream degraded → the poll is the page's rebuild path again.
|
||||
_liveCache.Live = false;
|
||||
var result = await sut.GetSiteAlarmsAsync(SiteId);
|
||||
|
||||
// Live deltas own the rows; only the not-reporting list still comes from the
|
||||
// fan-out, so a 30s-old answer is fine and costs no second fan-out.
|
||||
await _instanceRepo.Received(1).GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>());
|
||||
|
||||
_now = T0.AddSeconds(61);
|
||||
await sut.GetSiteAlarmsAsync(SiteId);
|
||||
await _instanceRepo.Received(2).GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>());
|
||||
Assert.Single(result.Alarms);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -113,7 +142,7 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
_instanceRepo.GetInstancesBySiteIdAsync(otherSite, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
|
||||
var sut = CreateSut(TimeSpan.FromSeconds(60));
|
||||
var sut = CreateSut();
|
||||
|
||||
await sut.GetSiteAlarmsAsync(SiteId);
|
||||
await sut.GetSiteAlarmsAsync(otherSite);
|
||||
@@ -125,7 +154,7 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
[Fact]
|
||||
public void PureMethods_MatchTheDirectImplementation()
|
||||
{
|
||||
var sut = CreateSut(TimeSpan.FromSeconds(60));
|
||||
var sut = CreateSut();
|
||||
var direct = new AlarmSummaryService(_instanceRepo, _siteRepo, _snapshotClient);
|
||||
var alarms = new List<AlarmStateChanged>
|
||||
{
|
||||
@@ -149,18 +178,24 @@ public class SharedAlarmSummaryServiceTests : IDisposable
|
||||
|
||||
public void Dispose() => _provider.Dispose();
|
||||
|
||||
/// <summary>Liveness-only stub — the façade consults nothing else on the live cache.</summary>
|
||||
/// <summary>
|
||||
/// Read-side stub: liveness, the published alarm snapshot, and the aggregator's
|
||||
/// not-reporting set — the three things the façade reads while the cache is serving a site.
|
||||
/// </summary>
|
||||
private sealed class FakeLiveCache : ISiteAlarmLiveCache
|
||||
{
|
||||
public bool Live { get; set; }
|
||||
public IReadOnlyList<AlarmStateChanged> Current { get; set; } = Array.Empty<AlarmStateChanged>();
|
||||
public IReadOnlyList<string> NotReporting { get; set; } = Array.Empty<string>();
|
||||
|
||||
public IDisposable Subscribe(int siteId, Action onChanged) => new NoOp();
|
||||
|
||||
public IReadOnlyList<AlarmStateChanged> GetCurrentAlarms(int siteId) =>
|
||||
Array.Empty<AlarmStateChanged>();
|
||||
public IReadOnlyList<AlarmStateChanged> GetCurrentAlarms(int siteId) => Current;
|
||||
|
||||
public bool IsLive(int siteId) => Live;
|
||||
|
||||
public IReadOnlyList<string> GetNotReportingInstances(int siteId) => NotReporting;
|
||||
|
||||
private sealed class NoOp : IDisposable
|
||||
{
|
||||
public void Dispose() { }
|
||||
|
||||
-151
@@ -1,151 +0,0 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for the Audit Log (#23) site→central ingest routing on
|
||||
/// <see cref="CentralCommunicationActor"/>. A site delivers
|
||||
/// <see cref="IngestAuditEventsCommand"/> / <see cref="IngestCachedTelemetryCommand"/>
|
||||
/// to the actor, which forwards to the registered
|
||||
/// <c>AuditLogIngestActor</c> proxy and routes the reply back to the site.
|
||||
/// Mirrors the NotificationSubmit / RegisterNotificationOutbox pattern.
|
||||
/// </summary>
|
||||
public class CentralCommunicationActorAuditTests : TestKit
|
||||
{
|
||||
public CentralCommunicationActorAuditTests() : base(@"akka.loglevel = DEBUG") { }
|
||||
|
||||
private IActorRef CreateActor(TimeSpan? auditIngestAskTimeout = null)
|
||||
{
|
||||
var mockRepo = Substitute.For<ISiteRepository>();
|
||||
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Commons.Entities.Sites.Site>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => mockRepo);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
return Sys.ActorOf(Props.Create(() =>
|
||||
new CentralCommunicationActor(sp, transport, auditIngestAskTimeout)));
|
||||
}
|
||||
|
||||
// C3 (Task 2.5): canonical ZB.MOM.WW.Audit.AuditEvent via the shared factory.
|
||||
private static AuditEvent SampleAuditEvent() =>
|
||||
ScadaBridgeAuditEventFactory.Create(
|
||||
channel: AuditChannel.ApiOutbound,
|
||||
kind: AuditKind.ApiCall,
|
||||
status: AuditStatus.Delivered);
|
||||
|
||||
private static SiteCall SampleSiteCall() => new()
|
||||
{
|
||||
TrackedOperationId = TrackedOperationId.New(),
|
||||
Channel = "OutboundApi",
|
||||
Target = "ExternalSystemA",
|
||||
SourceSite = "site1",
|
||||
Status = "Delivered",
|
||||
RetryCount = 0,
|
||||
CreatedAtUtc = DateTime.UtcNow,
|
||||
UpdatedAtUtc = DateTime.UtcNow,
|
||||
IngestedAtUtc = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEventsCommand_WithRegisteredProxy_ForwardsAndRoutesReplyToSender()
|
||||
{
|
||||
var actor = CreateActor();
|
||||
var auditProbe = CreateTestProbe();
|
||||
actor.Tell(new RegisterAuditIngest(auditProbe.Ref));
|
||||
|
||||
var evt = SampleAuditEvent();
|
||||
var cmd = new IngestAuditEventsCommand(new[] { evt });
|
||||
actor.Tell(cmd);
|
||||
|
||||
// The audit-ingest proxy receives the command, with the original site
|
||||
// sender preserved (Forward semantics).
|
||||
auditProbe.ExpectMsg(cmd);
|
||||
|
||||
// When the proxy replies, the actor routes it back to the original sender.
|
||||
var reply = new IngestAuditEventsReply(new[] { evt.EventId });
|
||||
auditProbe.Reply(reply);
|
||||
|
||||
var received = ExpectMsg<IngestAuditEventsReply>();
|
||||
Assert.Equal(new[] { evt.EventId }, received.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEventsCommand_WithNoProxyRegistered_RepliesEmptyAcceptedEventIds()
|
||||
{
|
||||
var actor = CreateActor();
|
||||
|
||||
actor.Tell(new IngestAuditEventsCommand(new[] { SampleAuditEvent() }));
|
||||
|
||||
var reply = ExpectMsg<IngestAuditEventsReply>();
|
||||
Assert.Empty(reply.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestAuditEventsCommand_WhenProxyNeverReplies_PipesStatusFailureToSender()
|
||||
{
|
||||
// A short test-only Ask timeout (constructor seam) keeps the test fast —
|
||||
// production uses the 30 s default.
|
||||
var actor = CreateActor(auditIngestAskTimeout: TimeSpan.FromMilliseconds(200));
|
||||
var auditProbe = CreateTestProbe();
|
||||
actor.Tell(new RegisterAuditIngest(auditProbe.Ref));
|
||||
|
||||
var cmd = new IngestAuditEventsCommand(new[] { SampleAuditEvent() });
|
||||
actor.Tell(cmd);
|
||||
|
||||
// The proxy receives the command but deliberately never replies.
|
||||
auditProbe.ExpectMsg(cmd);
|
||||
|
||||
// The Ask times out; PipeTo forwards the faulted task as a Status.Failure
|
||||
// to the original sender. This is the real transient signal the site's
|
||||
// own Ask faults on — it is NOT swallowed into an empty ack.
|
||||
var failure = ExpectMsg<Status.Failure>();
|
||||
Assert.IsType<AskTimeoutException>(failure.Cause);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestCachedTelemetryCommand_WithRegisteredProxy_ForwardsAndRoutesReplyToSender()
|
||||
{
|
||||
var actor = CreateActor();
|
||||
var auditProbe = CreateTestProbe();
|
||||
actor.Tell(new RegisterAuditIngest(auditProbe.Ref));
|
||||
|
||||
var entry = new CachedTelemetryEntry(SampleAuditEvent(), SampleSiteCall());
|
||||
var cmd = new IngestCachedTelemetryCommand(new[] { entry });
|
||||
actor.Tell(cmd);
|
||||
|
||||
auditProbe.ExpectMsg(cmd);
|
||||
|
||||
var reply = new IngestCachedTelemetryReply(new[] { entry.Audit.EventId });
|
||||
auditProbe.Reply(reply);
|
||||
|
||||
var received = ExpectMsg<IngestCachedTelemetryReply>();
|
||||
Assert.Equal(new[] { entry.Audit.EventId }, received.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void IngestCachedTelemetryCommand_WithNoProxyRegistered_RepliesEmptyAcceptedEventIds()
|
||||
{
|
||||
var actor = CreateActor();
|
||||
|
||||
var entry = new CachedTelemetryEntry(SampleAuditEvent(), SampleSiteCall());
|
||||
actor.Tell(new IngestCachedTelemetryCommand(new[] { entry }));
|
||||
|
||||
var reply = ExpectMsg<IngestCachedTelemetryReply>();
|
||||
Assert.Empty(reply.AcceptedEventIds);
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -37,7 +37,7 @@ public class CentralCommunicationActorClientLifecycleTests : TestKit
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(
|
||||
provider, transport, (TimeSpan?)null)));
|
||||
provider, transport)));
|
||||
|
||||
// Trigger the refresh (also fires at PreStart, but drive it explicitly so
|
||||
// the assertion is deterministic). The load runs on a detached task and
|
||||
|
||||
+1
-1
@@ -63,7 +63,7 @@ public class CentralCommunicationActorReconcileTests : TestKit
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
|
||||
// Node B is missing inst-B entirely → it should come back as a gap item.
|
||||
actor.Tell(new ReconcileSiteRequest(
|
||||
|
||||
@@ -40,7 +40,7 @@ public class CentralCommunicationActorTests : TestKit
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
return (actor, mockRepo);
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@ public class CentralCommunicationActorTests : TestKit
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var centralActor = Sys.ActorOf(
|
||||
Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
|
||||
var timestamp = DateTimeOffset.UtcNow;
|
||||
centralActor.Tell(new HeartbeatMessage("site1", "host1", true, timestamp));
|
||||
@@ -87,7 +87,7 @@ public class CentralCommunicationActorTests : TestKit
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var centralActor = Sys.ActorOf(
|
||||
Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
|
||||
var ts = DateTimeOffset.UtcNow;
|
||||
centralActor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage("site-1", "host-a", true, ts)));
|
||||
@@ -116,7 +116,7 @@ public class CentralCommunicationActorTests : TestKit
|
||||
// The fix logs a Warning carrying the InvalidOperationException as the cause.
|
||||
EventFilter.Warning(contains: "Failed to load site addresses from the database").ExpectOne(() =>
|
||||
{
|
||||
Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -40,7 +40,7 @@ public class CentralCommunicationActorTransportTests : TestKit
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
return (actor, transport, repo);
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,59 @@ public class CommunicationOptionsValidatorTests
|
||||
Assert.Contains("DeploymentTimeout", result.FailureMessage);
|
||||
}
|
||||
|
||||
// ── Audit-ingest timeout ladder (arch-review phase-2 residual #2) ────────────
|
||||
|
||||
[Fact]
|
||||
public void TheAuditIngestTimeoutLadder_IsStrictlyMonotonic_EndToEnd()
|
||||
{
|
||||
// 35 (site forward Ask) > 30 (gRPC deadline AND central's Ask of the ingest singleton)
|
||||
// > 20 (actor budget) > 15 (SQL command). Ties are the bug this closes: the site Ask
|
||||
// used to reuse NotificationForwardTimeout (30 s), so a slow-but-succeeding central
|
||||
// write could be acked to a caller that had already given up and re-sent the batch.
|
||||
var options = new CommunicationOptions();
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(35), options.AuditForwardTimeout);
|
||||
Assert.True(options.AuditForwardTimeout > ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout);
|
||||
Assert.Equal(TimeSpan.FromSeconds(30), ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AuditForwardTimeout_EqualToTheAskTimeout_IsRejected()
|
||||
{
|
||||
// Equality is precisely the pre-fix state, so the validator must refuse it, not just
|
||||
// refuse something smaller.
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
AuditForwardTimeout = ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout,
|
||||
});
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("AuditForwardTimeout", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AuditForwardTimeout_ShorterThanTheAskTimeout_IsRejected()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
AuditForwardTimeout = TimeSpan.FromSeconds(5),
|
||||
});
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains("AuditForwardTimeout", result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AuditForwardTimeout_IsStillConfigurableUpwards()
|
||||
{
|
||||
var result = Validate(new CommunicationOptions
|
||||
{
|
||||
AuditForwardTimeout = TimeSpan.FromMinutes(2),
|
||||
});
|
||||
|
||||
Assert.True(result.Succeeded, result.FailureMessage);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NonPositiveGrpcMaxConcurrentStreams_IsRejected()
|
||||
{
|
||||
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Audit Log (#23) site→central ingest routing on <see cref="CentralControlGrpcService"/>.
|
||||
/// <para>
|
||||
/// The service Asks the <c>audit-log-ingest</c> singleton proxy DIRECTLY. It used to relay
|
||||
/// through <see cref="CentralCommunicationActor"/>, which re-Asked the same proxy with the same
|
||||
/// 30 s <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/> — a second hop whose inner Ask
|
||||
/// expired at the same instant as the outer one, so it could only add latency. These tests pin
|
||||
/// the direct dispatch, the wiring-race reply, and the fact that the relay is gone.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class CentralControlGrpcServiceAuditIngestTests : TestKit
|
||||
{
|
||||
private static ServerCallContext NewContext(CancellationToken ct = default)
|
||||
{
|
||||
var context = Substitute.For<ServerCallContext>();
|
||||
context.CancellationToken.Returns(ct);
|
||||
return context;
|
||||
}
|
||||
|
||||
private static CentralControlGrpcService CreateService() => new(
|
||||
NullLogger<CentralControlGrpcService>.Instance,
|
||||
Options.Create(new CommunicationOptions()));
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_AsksTheIngestProxy_AndNeverTheCommunicationActor()
|
||||
{
|
||||
var ingest = CreateTestProbe();
|
||||
var control = CreateTestProbe();
|
||||
var service = CreateService();
|
||||
service.SetReady(control.Ref);
|
||||
service.SetAuditIngestActor(ingest.Ref);
|
||||
|
||||
var evt = SampleAuditEvent();
|
||||
var batch = new AuditEventBatch();
|
||||
batch.Events.Add(AuditEventDtoMapper.ToDto(evt));
|
||||
|
||||
var call = service.IngestAuditEvents(batch, NewContext());
|
||||
|
||||
var received = ingest.ExpectMsg<IngestAuditEventsCommand>();
|
||||
Assert.Equal(evt.EventId, Assert.Single(received.Events).EventId);
|
||||
ingest.Reply(new IngestAuditEventsReply(new[] { evt.EventId }));
|
||||
|
||||
var ack = await call;
|
||||
Assert.Equal(evt.EventId.ToString(), Assert.Single(ack.AcceptedEventIds));
|
||||
|
||||
// The old relay hop is gone: the control-plane actor sees nothing at all.
|
||||
control.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestCachedTelemetry_AsksTheIngestProxy_AndNeverTheCommunicationActor()
|
||||
{
|
||||
var ingest = CreateTestProbe();
|
||||
var control = CreateTestProbe();
|
||||
var service = CreateService();
|
||||
service.SetReady(control.Ref);
|
||||
service.SetAuditIngestActor(ingest.Ref);
|
||||
|
||||
var evt = SampleAuditEvent();
|
||||
var batch = new CachedTelemetryBatch();
|
||||
batch.Packets.Add(new CachedTelemetryPacket
|
||||
{
|
||||
AuditEvent = AuditEventDtoMapper.ToDto(evt),
|
||||
Operational = SiteCallDtoMapper.ToDto(SampleSiteCall()),
|
||||
});
|
||||
|
||||
var call = service.IngestCachedTelemetry(batch, NewContext());
|
||||
|
||||
var received = ingest.ExpectMsg<IngestCachedTelemetryCommand>();
|
||||
Assert.Single(received.Entries);
|
||||
ingest.Reply(new IngestCachedTelemetryReply(new[] { evt.EventId }));
|
||||
|
||||
var ack = await call;
|
||||
Assert.Equal(evt.EventId.ToString(), Assert.Single(ack.AcceptedEventIds));
|
||||
control.ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_BeforeTheIngestProxyIsWired_ReturnsAnEmptyAck()
|
||||
{
|
||||
// The singleton starts moments after SetReady, so this window is real. An empty ack
|
||||
// (NOT a fault, NOT Unavailable) leaves the site's rows Pending for the next drain —
|
||||
// byte-for-byte what the removed relay replied when its proxy was still null.
|
||||
var service = CreateService();
|
||||
service.SetReady(CreateTestProbe().Ref);
|
||||
|
||||
var batch = new AuditEventBatch();
|
||||
batch.Events.Add(AuditEventDtoMapper.ToDto(SampleAuditEvent()));
|
||||
|
||||
var ack = await service.IngestAuditEvents(batch, NewContext());
|
||||
|
||||
Assert.Empty(ack.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestCachedTelemetry_BeforeTheIngestProxyIsWired_ReturnsAnEmptyAck()
|
||||
{
|
||||
var service = CreateService();
|
||||
service.SetReady(CreateTestProbe().Ref);
|
||||
|
||||
var batch = new CachedTelemetryBatch();
|
||||
batch.Packets.Add(new CachedTelemetryPacket
|
||||
{
|
||||
AuditEvent = AuditEventDtoMapper.ToDto(SampleAuditEvent()),
|
||||
Operational = SiteCallDtoMapper.ToDto(SampleSiteCall()),
|
||||
});
|
||||
|
||||
var ack = await service.IngestCachedTelemetry(batch, NewContext());
|
||||
|
||||
Assert.Empty(ack.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SetAuditIngestActor_IsIndependentOfSetReady()
|
||||
{
|
||||
var service = CreateService();
|
||||
Assert.False(service.IsAuditIngestBound);
|
||||
|
||||
service.SetAuditIngestActor(CreateTestProbe().Ref);
|
||||
|
||||
Assert.True(service.IsAuditIngestBound);
|
||||
Assert.False(service.IsReady);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CentralCommunicationActor_NoLongerRelaysIngestCommands()
|
||||
{
|
||||
// Regression pin for the removed hop: the actor has no ingest receive at all, so an
|
||||
// ingest command reaching it is an unhandled message with no reply — not a silent
|
||||
// second path that could drift from the direct one.
|
||||
var actor = CreateCentralCommunicationActor();
|
||||
|
||||
actor.Tell(new IngestAuditEventsCommand(new[] { SampleAuditEvent() }), TestActor);
|
||||
actor.Tell(
|
||||
new IngestCachedTelemetryCommand(
|
||||
new[] { new CachedTelemetryEntry(SampleAuditEvent(), SampleSiteCall()) }),
|
||||
TestActor);
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
}
|
||||
|
||||
private IActorRef CreateCentralCommunicationActor()
|
||||
{
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Commons.Entities.Sites.Site>());
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
return Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
}
|
||||
|
||||
private static AuditEvent SampleAuditEvent() =>
|
||||
ScadaBridgeAuditEventFactory.Create(
|
||||
channel: AuditChannel.ApiOutbound,
|
||||
kind: AuditKind.ApiCall,
|
||||
status: AuditStatus.Delivered,
|
||||
sourceSiteId: "site-a");
|
||||
|
||||
private static SiteCall SampleSiteCall() => new()
|
||||
{
|
||||
TrackedOperationId = TrackedOperationId.New(),
|
||||
Channel = "OutboundApi",
|
||||
Target = "ExternalSystemA",
|
||||
SourceSite = "site-a",
|
||||
Status = "Delivered",
|
||||
RetryCount = 0,
|
||||
CreatedAtUtc = DateTime.UtcNow,
|
||||
UpdatedAtUtc = DateTime.UtcNow,
|
||||
IngestedAtUtc = DateTime.UtcNow,
|
||||
};
|
||||
}
|
||||
@@ -55,7 +55,7 @@ public class HealthReportAckTests : TestKit
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var transport = Substitute.For<ISiteCommandTransport>();
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
|
||||
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport)));
|
||||
|
||||
actor.Tell(SampleReport(seq: 3));
|
||||
var ack = ExpectMsg<SiteHealthReportAck>();
|
||||
|
||||
@@ -81,7 +81,7 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
|
||||
}
|
||||
|
||||
private SiteAlarmLiveCacheService CreateService(TimeSpan linger, out CountingFactory factory,
|
||||
int maxSubscribersPerSite = 200)
|
||||
int maxSubscribersPerSite = 200, IReadOnlyList<Instance>? enabledInstances = null)
|
||||
{
|
||||
// Site with gRPC addresses, and NO enabled instances → the seed fan-out returns
|
||||
// empty immediately (so IsLive flips true fast without any snapshot Asks).
|
||||
@@ -97,7 +97,7 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
|
||||
|
||||
var instanceRepo = Substitute.For<ITemplateEngineRepository>();
|
||||
instanceRepo.GetInstancesBySiteIdAsync(SiteId, Arg.Any<CancellationToken>())
|
||||
.Returns(new List<Instance>());
|
||||
.Returns((IReadOnlyList<Instance>)(enabledInstances ?? new List<Instance>()));
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
@@ -121,6 +121,45 @@ public class SiteAlarmLiveCacheServiceTests : TestKit
|
||||
return service;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Seed_FanOut_Publishes_The_Instances_That_Failed_To_Answer()
|
||||
{
|
||||
// Arch-review phase-2 residual #4: the seed/reconcile fan-out already knows which
|
||||
// Enabled instances failed to answer and used to discard it, forcing the Alarm Summary
|
||||
// page to run a SECOND identical fan-out purely to rebuild that list. It is now
|
||||
// published alongside the snapshot.
|
||||
var instances = new List<Instance>
|
||||
{
|
||||
new("inst-b") { Id = 2, SiteId = SiteId, State = InstanceState.Enabled },
|
||||
new("inst-a") { Id = 1, SiteId = SiteId, State = InstanceState.Enabled },
|
||||
// Disabled instances are never fanned out, so they can never be "not reporting".
|
||||
new("inst-off") { Id = 3, SiteId = SiteId, State = InstanceState.Disabled },
|
||||
};
|
||||
|
||||
// The CommunicationService has no site actor wired, so every snapshot Ask faults —
|
||||
// which is exactly the "instance did not answer" case.
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _, enabledInstances: instances);
|
||||
|
||||
using var sub = service.Subscribe(SiteId, () => { });
|
||||
AwaitCondition(() => service.IsLive(SiteId), TimeSpan.FromSeconds(5));
|
||||
|
||||
AwaitCondition(
|
||||
() => service.GetNotReportingInstances(SiteId).Count == 2,
|
||||
TimeSpan.FromSeconds(5));
|
||||
|
||||
// Ordered by name (ordinal-ignore-case), matching AlarmSummaryService's poll output so
|
||||
// the page renders identically whichever source supplied the list.
|
||||
Assert.Equal(new[] { "inst-a", "inst-b" }, service.GetNotReportingInstances(SiteId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NotReporting_Is_Empty_For_An_Unknown_Site()
|
||||
{
|
||||
var service = CreateService(TimeSpan.FromMilliseconds(200), out _);
|
||||
|
||||
Assert.Empty(service.GetNotReportingInstances(SiteId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void First_Subscriber_Starts_One_Aggregator_Shared_By_Multiple_Viewers()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
using System.Buffers.Binary;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Arch-review phase-2 residual #3: <see cref="CentralChannelProvider"/>'s failback probe reuses
|
||||
/// the <c>Heartbeat</c> RPC to ask "does the preferred central endpoint answer again?". It is
|
||||
/// emitted by a site's TRANSPORT layer, not by any node's heartbeat timer, so central must not
|
||||
/// count it as liveness — otherwise a site whose real heartbeats had stopped keeps looking alive
|
||||
/// on the health dashboard for as long as its transport keeps probing.
|
||||
/// <para>
|
||||
/// The contract is the explicit additive <c>Synthetic</c> flag (<c>HeartbeatDto.synthetic</c>,
|
||||
/// proto field 5), NOT the <c>failback-probe</c> hostname, which is a log label only.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class SyntheticHeartbeatTests : TestKit
|
||||
{
|
||||
private static readonly DateTimeOffset T0 =
|
||||
new(2026, 8, 14, 9, 0, 0, TimeSpan.Zero);
|
||||
|
||||
// ── The consumer: central skips liveness bookkeeping for a synthetic heartbeat ───
|
||||
|
||||
[Fact]
|
||||
public void SyntheticHeartbeat_DoesNotMarkTheHealthAggregator()
|
||||
{
|
||||
var (actor, aggregator) = CreateCentralActor();
|
||||
|
||||
actor.Tell(new HeartbeatMessage("site-1", CentralChannelProvider.SyntheticProbeHostname,
|
||||
IsActive: false, Timestamp: T0, Synthetic: true));
|
||||
|
||||
// Give the actor a real chance to (wrongly) mark before asserting the negative.
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RealHeartbeat_StillMarksTheHealthAggregator()
|
||||
{
|
||||
// The guard must be keyed on the flag alone — an ordinary heartbeat is unaffected,
|
||||
// including one from a node that predates the field (proto3 defaults it to false).
|
||||
var (actor, aggregator) = CreateCentralActor();
|
||||
|
||||
actor.Tell(new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0));
|
||||
|
||||
AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site-1", T0));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SyntheticHeartbeatReplica_IsAlsoSkipped()
|
||||
{
|
||||
// Belt-and-braces on the last hop before the aggregator: a peer central node that
|
||||
// predates the flag could still replicate one.
|
||||
var (actor, aggregator) = CreateCentralActor();
|
||||
|
||||
actor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage(
|
||||
"site-1", CentralChannelProvider.SyntheticProbeHostname,
|
||||
IsActive: false, Timestamp: T0, Synthetic: true)));
|
||||
|
||||
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
||||
aggregator.DidNotReceiveWithAnyArgs().MarkHeartbeat(default!, default);
|
||||
}
|
||||
|
||||
// ── The wire contract: the flag survives the round trip ─────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public void TheSyntheticFlag_RoundTripsThroughTheDto(bool synthetic)
|
||||
{
|
||||
var msg = new HeartbeatMessage("site-1", "node-a", IsActive: true, Timestamp: T0,
|
||||
Synthetic: synthetic);
|
||||
|
||||
var back = CentralControlDtoMapper.FromDto(CentralControlDtoMapper.ToDto(msg));
|
||||
|
||||
Assert.Equal(synthetic, back.Synthetic);
|
||||
Assert.Equal(msg with { Synthetic = synthetic }, back);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ADtoFromAnOlderSite_DefaultsToNotSynthetic()
|
||||
{
|
||||
// proto3 default: a peer that never sets field 5 sends a REAL heartbeat, as before.
|
||||
var dto = new HeartbeatDto
|
||||
{
|
||||
SiteId = "site-1",
|
||||
NodeHostname = "node-a",
|
||||
IsActive = true,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(T0),
|
||||
};
|
||||
|
||||
Assert.False(CentralControlDtoMapper.FromDto(dto).Synthetic);
|
||||
}
|
||||
|
||||
// ── The producer: the failback probe marks itself synthetic ─────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TheFailbackProbe_MarksItsHeartbeatSynthetic()
|
||||
{
|
||||
// Two endpoints so a flip is possible; the capture handler answers nothing useful, so
|
||||
// the probe faults and re-arms — we only care about the request it put on the wire.
|
||||
var capture = new HeartbeatCapturingHandler();
|
||||
using var provider = new CentralChannelProvider(
|
||||
new[] { "http://central-a:8083", "http://central-b:8083" },
|
||||
new FixedPskProvider("k"),
|
||||
"site-1",
|
||||
new CommunicationOptions(),
|
||||
NullLogger.Instance,
|
||||
handlerFactory: _ => capture,
|
||||
probeDeadline: TimeSpan.FromSeconds(2),
|
||||
backoffBase: TimeSpan.FromMilliseconds(20),
|
||||
backoffCap: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// Off the preferred endpoint → the background failback probe arms.
|
||||
provider.ReportUnavailable(0);
|
||||
|
||||
var probe = await capture.WaitForHeartbeatAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.True(probe.Synthetic);
|
||||
Assert.Equal("site-1", probe.SiteId);
|
||||
// The hostname is a human-readable label that rides ALONGSIDE the flag; central keys
|
||||
// its skip on the flag, never on this string.
|
||||
Assert.Equal(CentralChannelProvider.SyntheticProbeHostname, probe.NodeHostname);
|
||||
Assert.False(probe.IsActive);
|
||||
}
|
||||
|
||||
private (IActorRef Actor, ICentralHealthAggregator Aggregator) CreateCentralActor()
|
||||
{
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>()).Returns(new List<Site>());
|
||||
|
||||
var aggregator = Substitute.For<ICentralHealthAggregator>();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => siteRepo);
|
||||
services.AddSingleton(aggregator);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var actor = Sys.ActorOf(Props.Create(() =>
|
||||
new CentralCommunicationActor(sp, Substitute.For<ISiteCommandTransport>())));
|
||||
return (actor, aggregator);
|
||||
}
|
||||
|
||||
private sealed class FixedPskProvider(string key) : ISitePskProvider
|
||||
{
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct) => new(key);
|
||||
public void Invalidate(string siteId) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Captures the first <c>Heartbeat</c> request body and decodes the length-prefixed gRPC
|
||||
/// frame back into a <see cref="HeartbeatDto"/>. The response is deliberately a bare 500 so
|
||||
/// the probe treats the endpoint as still down; the provider swallows that and re-arms.
|
||||
/// </summary>
|
||||
private sealed class HeartbeatCapturingHandler : HttpMessageHandler
|
||||
{
|
||||
private readonly TaskCompletionSource<HeartbeatDto> _captured =
|
||||
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public async Task<HeartbeatDto> WaitForHeartbeatAsync(TimeSpan timeout)
|
||||
{
|
||||
var completed = await Task.WhenAny(_captured.Task, Task.Delay(timeout));
|
||||
Assert.Same(_captured.Task, completed);
|
||||
return await _captured.Task;
|
||||
}
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
if (request.Content is not null &&
|
||||
request.RequestUri?.AbsolutePath.EndsWith("/Heartbeat", StringComparison.Ordinal) == true)
|
||||
{
|
||||
var body = await request.Content.ReadAsByteArrayAsync(cancellationToken);
|
||||
// gRPC frame: 1 compression byte + 4-byte big-endian length + payload.
|
||||
if (body.Length >= 5)
|
||||
{
|
||||
var length = BinaryPrimitives.ReadInt32BigEndian(body.AsSpan(1, 4));
|
||||
_captured.TrySetResult(
|
||||
HeartbeatDto.Parser.ParseFrom(body.AsSpan(5, length).ToArray()));
|
||||
}
|
||||
}
|
||||
|
||||
return new HttpResponseMessage(System.Net.HttpStatusCode.InternalServerError)
|
||||
{
|
||||
Version = request.Version,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,10 @@ public class CentralControlEndToEndTests : IAsyncLifetime
|
||||
NullLogger<CentralControlGrpcService>.Instance,
|
||||
Options.Create(new CommunicationOptions()));
|
||||
_service.SetReady(stub);
|
||||
// The two ingest RPCs Ask the audit-ingest singleton DIRECTLY (no CentralCommunicationActor
|
||||
// relay any more), so the service needs its own proxy handed over. Reusing the same stub
|
||||
// keeps one actor answering both shapes.
|
||||
_service.SetAuditIngestActor(stub);
|
||||
|
||||
var pskProvider = new MapPskProvider(new Dictionary<string, string>
|
||||
{
|
||||
@@ -174,7 +178,7 @@ public class CentralControlEndToEndTests : IAsyncLifetime
|
||||
// ---- One RPC per shape: unary (above) + the ingest bridge ----
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_DecodesTheBatch_AsksTheActor_EncodesTheAck()
|
||||
public async Task IngestAuditEvents_DecodesTheBatch_AsksTheIngestProxy_EncodesTheAck()
|
||||
{
|
||||
var client = Client(SiteAKey, SiteA);
|
||||
|
||||
@@ -185,12 +189,13 @@ public class CentralControlEndToEndTests : IAsyncLifetime
|
||||
var ack = await client.IngestAuditEventsAsync(batch);
|
||||
|
||||
// The stub actor accepts one deterministic id per non-empty batch; its presence proves
|
||||
// the full DTO→command→Ask→reply→ack bridge ran, gated call and all.
|
||||
// the full DTO→command→Ask→reply→ack bridge ran straight to the ingest proxy, gated
|
||||
// call and all.
|
||||
Assert.Contains(AcceptedId.ToString(), ack.AcceptedEventIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task IngestAuditEvents_EmptyBatch_ShortCircuits_WithoutAskingTheActor()
|
||||
public async Task IngestAuditEvents_EmptyBatch_ShortCircuits_WithoutAskingTheIngestProxy()
|
||||
{
|
||||
// Even the empty-batch fast path is behind the gate — it still needs a valid key.
|
||||
var client = Client(SiteAKey, SiteA);
|
||||
@@ -238,9 +243,9 @@ public class CentralControlEndToEndTests : IAsyncLifetime
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Minimal stand-in for <c>CentralCommunicationActor</c>: answers the two RPC shapes this
|
||||
/// test exercises. Replies straight to the Ask's temp sender, exactly as the real actor's
|
||||
/// Forward/PipeTo paths do.
|
||||
/// Minimal stand-in for both actors the service Asks — <c>CentralCommunicationActor</c> for
|
||||
/// the unary control RPCs and the <c>audit-log-ingest</c> singleton for the ingest ones.
|
||||
/// Replies straight to the Ask's temp sender, exactly as the real actors do.
|
||||
/// </summary>
|
||||
private sealed class StubCentralActor : ReceiveActor
|
||||
{
|
||||
|
||||
@@ -59,12 +59,24 @@ public class SiteAuditPushFlowTests : TestKit
|
||||
private sealed class BridgeCentralTransport : ICentralTransport
|
||||
{
|
||||
private readonly IActorRef _central;
|
||||
public BridgeCentralTransport(IActorRef central) => _central = central;
|
||||
private readonly IActorRef _auditIngest;
|
||||
|
||||
/// <param name="central">Stands in for the central control plane's non-audit RPCs.</param>
|
||||
/// <param name="auditIngest">
|
||||
/// The central audit-ingest singleton. The two ingest RPCs go straight here, mirroring
|
||||
/// <c>CentralControlGrpcService</c>, which Asks the proxy directly rather than relaying
|
||||
/// through <c>CentralCommunicationActor</c>.
|
||||
/// </param>
|
||||
public BridgeCentralTransport(IActorRef central, IActorRef auditIngest)
|
||||
{
|
||||
_central = central;
|
||||
_auditIngest = auditIngest;
|
||||
}
|
||||
|
||||
public void SubmitNotification(NotificationSubmit message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void QueryNotificationStatus(NotificationStatusQuery message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void IngestAuditEvents(IngestAuditEventsCommand message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void IngestCachedTelemetry(IngestCachedTelemetryCommand message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void IngestAuditEvents(IngestAuditEventsCommand message, IActorRef replyTo) => _auditIngest.Tell(message, replyTo);
|
||||
public void IngestCachedTelemetry(IngestCachedTelemetryCommand message, IActorRef replyTo) => _auditIngest.Tell(message, replyTo);
|
||||
public void ReconcileSite(ReconcileSiteRequest message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void ReportSiteHealth(SiteHealthReport message, IActorRef replyTo) => _central.Tell(message, replyTo);
|
||||
public void SendHeartbeat(HeartbeatMessage message, IActorRef self) => _central.Tell(message, self);
|
||||
@@ -144,9 +156,11 @@ public class SiteAuditPushFlowTests : TestKit
|
||||
centralRepo,
|
||||
NullLogger<ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor>.Instance)));
|
||||
|
||||
// Real CentralCommunicationActor. Its periodic site-address refresh
|
||||
// resolves an ISiteRepository from this provider; an empty result keeps
|
||||
// the refresh a clean no-op and never touches the audit-ingest path.
|
||||
// Real CentralCommunicationActor. It is NOT on the audit path any more (the
|
||||
// central-hosted CentralControlGrpcService Asks the ingest singleton directly), but the
|
||||
// bridge transport still routes notifications/health/reconcile through it, so it is
|
||||
// constructed exactly as production does. Its periodic site-address refresh resolves an
|
||||
// ISiteRepository from this provider; an empty result keeps the refresh a clean no-op.
|
||||
var siteRepo = Substitute.For<ISiteRepository>();
|
||||
siteRepo.GetAllSitesAsync().Returns(Array.Empty<Site>());
|
||||
var centralServices = new ServiceCollection();
|
||||
@@ -155,9 +169,7 @@ public class SiteAuditPushFlowTests : TestKit
|
||||
|
||||
var centralCommActor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(
|
||||
centralProvider,
|
||||
Substitute.For<ISiteCommandTransport>(),
|
||||
TimeSpan.FromSeconds(5))));
|
||||
centralCommActor.Tell(new RegisterAuditIngest(ingestActor));
|
||||
Substitute.For<ISiteCommandTransport>())));
|
||||
|
||||
// ── Site side ─────────────────────────────────────────────────────
|
||||
// Real SqliteAuditWriter on a file-backed SQLite db (the site hot-path
|
||||
@@ -177,7 +189,7 @@ public class SiteAuditPushFlowTests : TestKit
|
||||
CreateTestProbe().Ref, // deployment-manager proxy is unused here
|
||||
null,
|
||||
null,
|
||||
new BridgeCentralTransport(centralCommActor))));
|
||||
new BridgeCentralTransport(centralCommActor, ingestActor))));
|
||||
|
||||
// The production site audit push client — the unit under integration.
|
||||
var auditClient = new SiteCommunicationAuditClient(
|
||||
|
||||
Reference in New Issue
Block a user