Joseph Doherty
0289d51137
A DeployedSecretsMirror failure on the STANDBY node (e.g. a master-key mismatch
that stops a deployed secret decrypting there) was only logged on that node:
site health reports are sent by the active node only, so central never saw it.
- New 12th LocalDb-replicated table secret_mirror_status (node_name PK,
failures_json of {SecretName, Message}, checked_at; no FKs). Each node's
mirror writes only its own row, and only when its failure set changes plus
once per start-up (every write replicates via CDC). Best-effort: a failed
write is a Warning and is retried next pass. Node key = INodeIdentityProvider
NodeName, falling back to the machine name.
- SecretMirrorStatusReporter (SiteRuntime, every node, 30 s) reads every row
into ISiteHealthCollector.SetSecretMirrorStatuses -> additive nullable
SiteHealthReport.SecretMirrorStatuses (null = no data).
- Wire: SiteHealthReportDto.secret_mirror_statuses = 33 (presence wrapper so
null != empty); vendored CentralControl.cs regenerated; mapper both ways.
- Central UI Health card: 'Secret mirror failing' badge + compact per-node
node / secret / message list.
- Tests: mirror status write-on-change/first-pass/failure isolation, storage
round-trip, reporter/collector, DTO mapper round-trip incl. null/empty,
bUnit, replicated-table set (twelve) + FK guard wording.
729 lines
29 KiB
C#
729 lines
29 KiB
C#
using System.Security.Claims;
|
|
using ZB.MOM.WW.ScadaBridge.Security;
|
|
using Akka.Actor;
|
|
using Bunit;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.AspNetCore.Components;
|
|
using Microsoft.AspNetCore.Components.Authorization;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Microsoft.Extensions.Options;
|
|
using NSubstitute;
|
|
using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared;
|
|
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
|
using ZB.MOM.WW.ScadaBridge.Communication;
|
|
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
|
using HealthPage = ZB.MOM.WW.ScadaBridge.CentralUI.Components.Pages.Monitoring.Health;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.Pages;
|
|
|
|
/// <summary>
|
|
/// bUnit rendering tests for the Health Monitoring dashboard (Task 24).
|
|
///
|
|
/// Scope: the Notification Outbox KPI tile row added to the Health dashboard.
|
|
/// <see cref="ICentralHealthAggregator"/> is an interface (mockable), but
|
|
/// <see cref="CommunicationService"/> is a concrete class whose outbox calls
|
|
/// route through an injected notification-outbox <see cref="IActorRef"/>; the
|
|
/// tests reuse the scripted-actor seam established by the Notification Report
|
|
/// page tests (see <c>NotificationReportPageTests</c>).
|
|
/// </summary>
|
|
public class HealthPageTests : BunitContext
|
|
{
|
|
private readonly ActorSystem _system = ActorSystem.Create("health-page-tests");
|
|
private readonly CommunicationService _comms;
|
|
|
|
// Mutable scripted reply — individual tests can override before rendering.
|
|
private NotificationKpiResponse _kpiReply =
|
|
new("k", true, null, QueueDepth: 12, StuckCount: 4, ParkedCount: 3,
|
|
DeliveredLastInterval: 88, OldestPendingAge: TimeSpan.FromMinutes(6));
|
|
|
|
// Site Call Audit (#22) Task 7 — mutable scripted Site Call KPI reply. Tests
|
|
// that target the Site Call tiles override this before rendering.
|
|
private SiteCallKpiResponse _siteCallKpiReply =
|
|
new("k", true, null, BufferedCount: 9, ParkedCount: 2, FailedLastInterval: 1,
|
|
DeliveredLastInterval: 40, OldestPendingAge: TimeSpan.FromMinutes(3),
|
|
StuckCount: 5);
|
|
|
|
public HealthPageTests()
|
|
{
|
|
_comms = new CommunicationService(
|
|
Options.Create(new CommunicationOptions()),
|
|
NullLogger<CommunicationService>.Instance);
|
|
|
|
var outbox = _system.ActorOf(Props.Create(() => new ScriptedOutboxActor(this)));
|
|
_comms.SetNotificationOutbox(outbox);
|
|
|
|
var siteCallAudit = _system.ActorOf(Props.Create(() => new ScriptedSiteCallAuditActor(this)));
|
|
_comms.SetSiteCallAudit(siteCallAudit);
|
|
Services.AddSingleton(_comms);
|
|
|
|
// arch-review WP2.4 — the page no longer queries CommunicationService /
|
|
// IAuditLogQueryService per circuit; it reads the process-level memoized
|
|
// KPI cache. The real cache is registered here (not a substitute) so the
|
|
// scripted-actor seam above is still what actually answers.
|
|
Services.AddSingleton<IKpiSnapshotCache>(sp => new KpiSnapshotCache(
|
|
sp.GetRequiredService<CommunicationService>(),
|
|
sp.GetRequiredService<IServiceScopeFactory>()));
|
|
|
|
var aggregator = Substitute.For<ICentralHealthAggregator>();
|
|
aggregator.GetAllSiteStates()
|
|
.Returns(new Dictionary<string, SiteHealthState>());
|
|
Services.AddSingleton(aggregator);
|
|
|
|
// M6 K16 — the Health page now injects IKpiHistoryQueryService to feed the
|
|
// per-site Site Health Trends panel. Stub it with a known non-empty series
|
|
// so the page resolves the dependency and the trend charts have data; the
|
|
// dedicated trend tests below seed sites / override behaviour.
|
|
var kpiHistory = Substitute.For<IKpiHistoryQueryService>();
|
|
kpiHistory.GetSeriesAsync(
|
|
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
|
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
|
|
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(SampleSeries()));
|
|
Services.AddSingleton(kpiHistory);
|
|
|
|
var siteRepo = Substitute.For<ISiteRepository>();
|
|
siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
|
|
.Returns(Task.FromResult<IReadOnlyList<Site>>(new List<Site>()));
|
|
Services.AddSingleton(siteRepo);
|
|
|
|
// PLAN-01 Task 14 — the Health page now shows the metrics-stale badge with
|
|
// a tooltip naming the configured MetricsStaleTimeout, so it injects the
|
|
// health-monitoring options.
|
|
Services.AddSingleton(Options.Create(new HealthMonitoringOptions()));
|
|
|
|
// Audit Log (#23) M7 Bundle E — the Health page now also fetches the
|
|
// Audit KPI snapshot. Stub it with an empty point-in-time reading so
|
|
// the existing assertions (Notification Outbox tiles, Online/Offline
|
|
// counts) keep passing; tests that target the Audit tiles set their
|
|
// own substitute.
|
|
var auditService = Substitute.For<IAuditLogQueryService>();
|
|
auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
|
|
.Returns(Task.FromResult(new AuditLogKpiSnapshot(0, 0, 0, DateTime.UtcNow)));
|
|
Services.AddSingleton(auditService);
|
|
|
|
var claims = new[]
|
|
{
|
|
new Claim(JwtTokenService.UsernameClaimType, "tester"),
|
|
new Claim(JwtTokenService.RoleClaimType, "Administrator"),
|
|
};
|
|
var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
|
|
Services.AddSingleton<AuthenticationStateProvider>(new TestAuthStateProvider(user));
|
|
Services.AddAuthorizationCore();
|
|
Services.AddTestConfigurationMutations();
|
|
// The failover control's AuthorizeView evaluates the named RequireAdmin policy, so the
|
|
// real policy set has to be registered or the page throws on any card that renders it.
|
|
AuthorizationPolicies.AddScadaBridgeAuthorization(Services);
|
|
// BunitContext pre-registers a placeholder IAuthorizationService that throws when a
|
|
// policy is evaluated; force the real one (same note as NavMenuTests).
|
|
Services.AddSingleton<IAuthorizationService, DefaultAuthorizationService>();
|
|
|
|
// The cluster cards embed CentralFailoverControl (Task 10), which injects these two.
|
|
// The page's DI graph genuinely requires them in production, so the test supplies
|
|
// them rather than the component making its dependencies optional. Behaviour of the
|
|
// control itself is covered by HealthFailoverButtonTests.
|
|
Services.AddSingleton(Substitute.For<IManualFailoverService>());
|
|
Services.AddSingleton(Substitute.For<IDialogService>());
|
|
}
|
|
|
|
/// <summary>
|
|
/// Renders the dashboard the way the app does — inside a
|
|
/// <see cref="CascadingAuthenticationState"/>, which the real layout supplies. The
|
|
/// cluster cards contain an AuthorizeView (the Task 10 failover control), and
|
|
/// AuthorizeView throws without that cascading value.
|
|
/// </summary>
|
|
private IRenderedComponent<HealthPage> RenderHealthPage()
|
|
{
|
|
var host = Render<CascadingAuthenticationState>(parameters => parameters
|
|
.Add(p => p.ChildContent, (RenderFragment)(builder =>
|
|
{
|
|
builder.OpenComponent<HealthPage>(0);
|
|
builder.CloseComponent();
|
|
})));
|
|
|
|
return host.FindComponent<HealthPage>();
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_OutboxKpiTiles_WithValues()
|
|
{
|
|
var cut = RenderHealthPage();
|
|
|
|
// KPI data arrives via an async actor Ask after first render.
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("Notification Outbox", cut.Markup);
|
|
Assert.Contains("Queue Depth", cut.Markup);
|
|
Assert.Contains("Stuck", cut.Markup);
|
|
Assert.Contains("Parked", cut.Markup);
|
|
// KPI numeric values surface in the tiles.
|
|
Assert.Contains(">12<", cut.Markup); // QueueDepth
|
|
Assert.Contains(">4<", cut.Markup); // StuckCount
|
|
Assert.Contains(">3<", cut.Markup); // ParkedCount
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void RendersLinkToTheNotificationKpisPage()
|
|
{
|
|
var cut = RenderHealthPage();
|
|
var link = cut.Find("a[href='/notifications/kpis']");
|
|
Assert.Contains("View details", link.TextContent);
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_AuditKpiTiles_WithValues()
|
|
{
|
|
// Override the default empty snapshot — this test wants concrete values
|
|
// to land in the three Audit tiles.
|
|
var auditService = Substitute.For<IAuditLogQueryService>();
|
|
auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
|
|
.Returns(Task.FromResult(new AuditLogKpiSnapshot(
|
|
TotalEventsLastHour: 250,
|
|
ErrorEventsLastHour: 5,
|
|
BacklogTotal: 17,
|
|
AsOfUtc: DateTime.UtcNow)));
|
|
Services.AddSingleton(auditService);
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
// The three audit tiles render at the documented data-test selectors.
|
|
Assert.Contains("data-test=\"audit-kpi-volume\"", cut.Markup);
|
|
Assert.Contains("data-test=\"audit-kpi-error-rate\"", cut.Markup);
|
|
Assert.Contains("data-test=\"audit-kpi-backlog\"", cut.Markup);
|
|
// Volume shows the formatted thousand-separator value.
|
|
Assert.Contains("250", cut.Markup);
|
|
// Backlog renders 17.
|
|
Assert.Contains("17", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_SiteCallKpiTiles_WithValues()
|
|
{
|
|
var cut = RenderHealthPage();
|
|
|
|
// KPI data arrives via an async actor Ask after first render.
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("Site Calls", cut.Markup);
|
|
// The three Site Call tiles render at the documented data-test selectors.
|
|
Assert.Contains("data-test=\"site-call-kpi-buffered\"", cut.Markup);
|
|
Assert.Contains("data-test=\"site-call-kpi-stuck\"", cut.Markup);
|
|
Assert.Contains("data-test=\"site-call-kpi-parked\"", cut.Markup);
|
|
// KPI numeric values surface in the tiles.
|
|
Assert.Contains(">9<", cut.Markup); // BufferedCount
|
|
Assert.Contains(">5<", cut.Markup); // StuckCount
|
|
Assert.Contains(">2<", cut.Markup); // ParkedCount
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void RendersLinkToTheSiteCallsReportPage()
|
|
{
|
|
var cut = RenderHealthPage();
|
|
var link = cut.Find("a[href='/site-calls/report']");
|
|
Assert.Contains("View details", link.TextContent);
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteCallKpiFailure_ShowsGracefulFallback()
|
|
{
|
|
_siteCallKpiReply = new SiteCallKpiResponse(
|
|
"k", false, "site call repository unavailable", 0, 0, 0, 0, null, 0);
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
// Failure must not crash the page; tiles fall back to a dash and the
|
|
// inline error message surfaces.
|
|
Assert.Contains("Site Calls", cut.Markup);
|
|
Assert.Contains("Site Call KPIs unavailable", cut.Markup);
|
|
Assert.Contains("site call repository unavailable", cut.Markup);
|
|
Assert.Contains(">—<", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void OutboxKpiFailure_ShowsGracefulFallback()
|
|
{
|
|
_kpiReply = new NotificationKpiResponse(
|
|
"k", false, "outbox repository unavailable", 0, 0, 0, 0, null);
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
// Failure must not crash the page; tiles fall back to a dash.
|
|
Assert.Contains("Notification Outbox", cut.Markup);
|
|
Assert.Contains("Queue Depth", cut.Markup);
|
|
Assert.Contains(">—<", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_SiteHealthTrends_PanelAndChart_ForSelectedSite()
|
|
{
|
|
// Seed one site so the trend panel's selector has an option and the
|
|
// default-site load produces charts.
|
|
SeedSites("site-a");
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
// The panel + its site selector render at the documented hooks.
|
|
Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
|
|
Assert.Contains("data-test=\"site-health-trends-site\"", cut.Markup);
|
|
// The four metric charts render (the shared KpiTrendChart slug hook),
|
|
// and the seeded non-empty series draws a polyline. The "S&F Buffer
|
|
// Depth" title slugifies to "s-f-buffer-depth" (the & and the spaces
|
|
// each collapse to a dash) — see KpiTrendChart.Slugify.
|
|
Assert.Contains("kpi-trend-connections-down", cut.Markup);
|
|
Assert.Contains("kpi-trend-dead-letters", cut.Markup);
|
|
Assert.Contains("kpi-trend-script-errors", cut.Markup);
|
|
Assert.Contains("kpi-trend-s-f-buffer-depth", cut.Markup);
|
|
Assert.Contains("<polyline", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void SiteHealthTrendsFailure_DoesNotBreakDashboard()
|
|
{
|
|
SeedSites("site-a");
|
|
|
|
// The KPI-history service throws on every query — the trend load is
|
|
// best-effort, so the dashboard (and its tiles) must still render.
|
|
var faulting = Substitute.For<IKpiHistoryQueryService>();
|
|
faulting.GetSeriesAsync(
|
|
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
|
|
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
|
|
.Returns<Task<IReadOnlyList<KpiSeriesPoint>>>(_ =>
|
|
throw new InvalidOperationException("kpi history unavailable"));
|
|
Services.AddSingleton(faulting);
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
// No unhandled exception: the core dashboard tiles still render, and
|
|
// the panel falls back to the per-chart unavailable placeholder.
|
|
Assert.Contains("Notification Outbox", cut.Markup);
|
|
Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
|
|
Assert.Contains("Trend data unavailable.", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_MetricsStaleBadge_ForOnlineButStaleSite()
|
|
{
|
|
// PLAN-01 Task 14: a site that is online (heartbeating) but whose report
|
|
// pipeline died shows a distinct "Metrics stale" warning badge.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
IsMetricsStale = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LastReportReceivedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("Metrics stale", cut.Markup);
|
|
// The badge uses the page's Bootstrap warning style.
|
|
Assert.Contains("bg-warning", cut.Markup);
|
|
});
|
|
}
|
|
|
|
// ------------------------------------------------------------------
|
|
// Sustained LocalDb replication backlog badge (register row 33)
|
|
// ------------------------------------------------------------------
|
|
|
|
[Fact]
|
|
public void Renders_ReplicationBacklogBadge_WhenBacklogSustainedPastTheDwell()
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
// Default dwell is 15 minutes; 30 minutes above threshold is well past it.
|
|
LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-30),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("data-test=\"localdb-backlog-badge\"", cut.Markup);
|
|
Assert.Contains("Replication backlog", cut.Markup);
|
|
});
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_ReplicationBacklogBadge_DuringAPurgeBurst()
|
|
{
|
|
// THE false-positive this badge is shaped to avoid. The daily site_events purge is
|
|
// CDC-captured, so the backlog spikes and then drains. Two minutes above the
|
|
// threshold is a purge burst, not a fault, and must stay silent — otherwise the
|
|
// badge fires on every replicated site every day and gets ignored.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-2),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_ReplicationBacklogBadge_ForAnUnreplicatedSite()
|
|
{
|
|
// Rig site-b/site-c report a null backlog forever, so the aggregator never gives them
|
|
// an anchor. A deliberately-unreplicated site must never be badged.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-b",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LocalDbBacklogAboveSince = null,
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-b", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_ReplicationBacklogBadge_ForAnOfflineSite()
|
|
{
|
|
// An offline site's last-known anchor is stale by definition; the Offline badge is the
|
|
// signal that matters, and stacking a backlog warning on top is noise.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = false,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
|
LocalDbBacklogAboveSince = DateTimeOffset.UtcNow.AddMinutes(-30),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"localdb-backlog-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_LocalDbBacklogTrendChart()
|
|
{
|
|
// The retrospective half of row 33: a point-in-time tile can never answer "did that
|
|
// spike drain?", which is the only question that distinguishes healthy from faulty.
|
|
SeedSites("site-a");
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
Assert.Contains("LocalDb Replication Backlog", cut.Markup));
|
|
}
|
|
|
|
// ----- Register row 73: the "No self-heal" badge -----------------------------------
|
|
//
|
|
// Tri-state, and only the explicit false is badged. Null means "not reported" — the
|
|
// central self-report row, and every site node on a pre-row-73 build — so badging null
|
|
// would light up the whole fleet the moment central is upgraded ahead of the sites.
|
|
|
|
[Fact]
|
|
public void Renders_StartupReconcilerBadge_WhenTheSiteReportsItIsMissing()
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = ReportWithReconciler("site-a", active: false),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
Assert.Contains("data-test=\"startup-reconciler-badge\"", cut.Markup));
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_StartupReconcilerBadge_WhenTheReconcilerIsRunning()
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = ReportWithReconciler("site-a", active: true),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_StartupReconcilerBadge_WhenNotReported()
|
|
{
|
|
// Null = "not reported", never "missing". This is central's own self-report row and
|
|
// any site that has not yet been upgraded.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = ReportWithReconciler("site-a", active: null),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_StartupReconcilerBadge_WhenTheSiteHasNoReportYet()
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = null,
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void DoesNotRender_StartupReconcilerBadge_ForAnOfflineSite()
|
|
{
|
|
// Same rule as the replication-backlog badge: the Offline badge is the signal that
|
|
// matters, and stacking a stale warning on top of it is noise.
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = false,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow.AddMinutes(-30),
|
|
LatestReport = ReportWithReconciler("site-a", active: false),
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"startup-reconciler-badge\"", cut.Markup);
|
|
}
|
|
|
|
private static SiteHealthReport ReportWithReconciler(string siteId, bool? active) =>
|
|
new(
|
|
SiteId: siteId,
|
|
SequenceNumber: 1,
|
|
ReportTimestamp: DateTimeOffset.UtcNow,
|
|
DataConnectionStatuses:
|
|
new Dictionary<string, Commons.Types.Enums.ConnectionHealth>(),
|
|
TagResolutionCounts: new Dictionary<string, TagResolutionStatus>(),
|
|
ScriptErrorCount: 0,
|
|
AlarmEvaluationErrorCount: 0,
|
|
StoreAndForwardBufferDepths: new Dictionary<string, int>(),
|
|
DeadLetterCount: 0,
|
|
DeployedInstanceCount: 0,
|
|
EnabledInstanceCount: 0,
|
|
DisabledInstanceCount: 0)
|
|
{
|
|
StartupReconcilerActive = active,
|
|
};
|
|
|
|
// ----- Review finding M1: per-node deployed-secrets mirror failures ------------------
|
|
//
|
|
// The standby's failures ride the active node's report; the card must name the node,
|
|
// the secret and the message. Null ("no data") and all-healthy nodes render nothing.
|
|
|
|
[Fact]
|
|
public void Renders_SecretMirrorFailures_LabelledByNode()
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = ReportWithReconciler("site-a", active: true) with
|
|
{
|
|
SecretMirrorStatuses =
|
|
[
|
|
new SecretMirrorNodeStatus("node-a", DateTimeOffset.UtcNow, []),
|
|
new SecretMirrorNodeStatus(
|
|
"node-b",
|
|
DateTimeOffset.UtcNow,
|
|
[new SecretMirrorFailure("OPC-PWD", "did not decrypt on this node")]),
|
|
],
|
|
},
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("data-test=\"secret-mirror-badge\"", cut.Markup);
|
|
var panel = cut.Find("[data-test=secret-mirror-failures]");
|
|
Assert.Contains("node-b", panel.TextContent);
|
|
Assert.Contains("OPC-PWD", panel.TextContent);
|
|
Assert.Contains("did not decrypt on this node", panel.TextContent);
|
|
Assert.DoesNotContain("node-a", panel.TextContent);
|
|
});
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData(false)] // null = "no data" (central self-report, pre-M1 site)
|
|
[InlineData(true)] // every node healthy
|
|
public void DoesNotRender_SecretMirrorWarning_WhenNoNodeIsFailing(bool reported)
|
|
{
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
LatestReport = ReportWithReconciler("site-a", active: true) with
|
|
{
|
|
SecretMirrorStatuses = reported
|
|
? [new SecretMirrorNodeStatus("node-a", DateTimeOffset.UtcNow, [])]
|
|
: null,
|
|
},
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() => Assert.Contains("site-a", cut.Markup));
|
|
Assert.DoesNotContain("data-test=\"secret-mirror-badge\"", cut.Markup);
|
|
Assert.DoesNotContain("data-test=\"secret-mirror-failures\"", cut.Markup);
|
|
}
|
|
|
|
[Fact]
|
|
public void Renders_OfflineSince_ForOfflineSiteWithStatusChange()
|
|
{
|
|
var changedAt = new DateTimeOffset(2026, 7, 8, 12, 34, 56, TimeSpan.Zero);
|
|
SeedStates(new SiteHealthState
|
|
{
|
|
SiteId = "site-a",
|
|
IsOnline = false,
|
|
LastHeartbeatAt = changedAt,
|
|
LastStatusChangeAt = changedAt,
|
|
});
|
|
|
|
var cut = RenderHealthPage();
|
|
|
|
cut.WaitForAssertion(() =>
|
|
{
|
|
Assert.Contains("offline since", cut.Markup);
|
|
// The "u" (universal sortable) format the page renders the flip with.
|
|
Assert.Contains(changedAt.ToString("u"), cut.Markup);
|
|
});
|
|
}
|
|
|
|
// Re-seeds the aggregator substitute with fully-specified states (as opposed
|
|
// to SeedSites' minimal online placeholders).
|
|
private void SeedStates(params SiteHealthState[] states)
|
|
{
|
|
var aggregator = Substitute.For<ICentralHealthAggregator>();
|
|
aggregator.GetAllSiteStates()
|
|
.Returns(states.ToDictionary(s => s.SiteId));
|
|
Services.AddSingleton(aggregator);
|
|
}
|
|
|
|
// Re-seeds the aggregator substitute so the trend panel's site selector has
|
|
// options. Each site id maps to a minimal online SiteHealthState (a null
|
|
// report is fine — the trend panel keys off the site ids, not the report).
|
|
private void SeedSites(params string[] siteIds)
|
|
{
|
|
var aggregator = Substitute.For<ICentralHealthAggregator>();
|
|
var states = siteIds.ToDictionary(
|
|
id => id,
|
|
id => new SiteHealthState
|
|
{
|
|
SiteId = id,
|
|
IsOnline = true,
|
|
LastHeartbeatAt = DateTimeOffset.UtcNow,
|
|
});
|
|
aggregator.GetAllSiteStates()
|
|
.Returns(new Dictionary<string, SiteHealthState>(states));
|
|
Services.AddSingleton(aggregator);
|
|
}
|
|
|
|
// A known non-empty (≥2-point) series so KpiTrendChart renders a polyline
|
|
// rather than the single-sample / unavailable placeholder.
|
|
private static IReadOnlyList<KpiSeriesPoint> SampleSeries()
|
|
{
|
|
var baseUtc = DateTime.UtcNow.AddHours(-24);
|
|
return new List<KpiSeriesPoint>
|
|
{
|
|
new(baseUtc, 1),
|
|
new(baseUtc.AddHours(6), 3),
|
|
new(baseUtc.AddHours(12), 2),
|
|
new(baseUtc.AddHours(18), 5),
|
|
};
|
|
}
|
|
|
|
protected override void Dispose(bool disposing)
|
|
{
|
|
if (disposing)
|
|
{
|
|
_system.Terminate().Wait(TimeSpan.FromSeconds(5));
|
|
}
|
|
base.Dispose(disposing);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stand-in for the notification-outbox actor. Replies to the KPI request
|
|
/// with the test's currently-scripted response.
|
|
/// </summary>
|
|
private sealed class ScriptedOutboxActor : ReceiveActor
|
|
{
|
|
public ScriptedOutboxActor(HealthPageTests test)
|
|
{
|
|
Receive<NotificationKpiRequest>(_ => Sender.Tell(test._kpiReply));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stand-in for the Site Call Audit actor. Replies to the KPI request with
|
|
/// the test's currently-scripted response. Also handles the per-node KPI
|
|
/// request (T6: M5.2) with an empty-nodes success reply so the Health page
|
|
/// can complete initialization without a 30-second Ask timeout.
|
|
/// </summary>
|
|
private sealed class ScriptedSiteCallAuditActor : ReceiveActor
|
|
{
|
|
public ScriptedSiteCallAuditActor(HealthPageTests test)
|
|
{
|
|
Receive<SiteCallKpiRequest>(_ => Sender.Tell(test._siteCallKpiReply));
|
|
Receive<PerNodeSiteCallKpiRequest>(req => Sender.Tell(
|
|
new PerNodeSiteCallKpiResponse(req.CorrelationId, Success: true, ErrorMessage: null,
|
|
Nodes: Array.Empty<SiteCallNodeKpiSnapshot>())));
|
|
}
|
|
}
|
|
}
|