Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.CentralUI.Tests/Pages/HealthPageTests.cs
T
Joseph Doherty c8e2f4da02 feat(cluster): site-pair manual failover relayed from the central UI (Task 10)
Central and each site are SEPARATE Akka clusters, so central cannot act on a
site's membership -- it asks. New TriggerSiteFailover/SiteFailoverAck contract
travels the existing ClusterClient command/control channel (mirroring the
RetryParkedOperation relay); the site's own SiteCommunicationActor performs the
graceful Leave and acks the outcome.

- ClusterFailoverCoordinator moved out of Host into Communication/ClusterState,
  beside ActiveNodeEvaluator. Both paths now share ONE oldest-Up implementation;
  SiteCommunicationActor cannot reference Host, and the two definitions must not
  drift or the node asked to leave stops being the singleton host.
- Site scope is the SITE-SPECIFIC role (site-{SiteId}), not the base Site role --
  site singletons are placed on the former, so the base role would move the wrong
  node. Pinned by a unit test asserting the role string and by a real-cluster test.
- Site-side guards: refuses a command addressed to another site (a misroute must
  never fail over a site the operator did not select), refuses when there is no
  peer, and reports a fault as an ack rather than throwing into supervision --
  a restart there would drop central's Ask into a bare timeout and lose the reason.
- Ack is sent before the Leave takes effect so it still reaches central.
- UI: the same control now serves both scopes via a SiteId parameter. The site
  confirmation deliberately does NOT claim the admin's page will disconnect --
  it won't, and crying wolf there devalues the central warning that is real. A
  site refusal and an unreachable site surface distinctly.
- Rolling upgrade: a site on an older binary has no handler, so the message
  dead-letters and the Ask times out, reported as "site did not respond". That
  is the honest outcome; documented on the contract.

Fallout fixed: HealthPageTests now renders the page inside
CascadingAuthenticationState with the real policy set and IAuthorizationService,
because the cards embed an AuthorizeView. That mirrors production, where the
layout supplies the cascading value.
2026-07-22 07:48:56 -04:00

444 lines
18 KiB
C#

using System.Security.Claims;
using ZB.MOM.WW.ScadaBridge.Security;
using Akka.Actor;
using Bunit;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.CentralUI.Components.Shared;
using ZB.MOM.WW.ScadaBridge.CentralUI.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
using HealthPage = ZB.MOM.WW.ScadaBridge.CentralUI.Components.Pages.Monitoring.Health;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Tests.Pages;
/// <summary>
/// bUnit rendering tests for the Health Monitoring dashboard (Task 24).
///
/// Scope: the Notification Outbox KPI tile row added to the Health dashboard.
/// <see cref="ICentralHealthAggregator"/> is an interface (mockable), but
/// <see cref="CommunicationService"/> is a concrete class whose outbox calls
/// route through an injected notification-outbox <see cref="IActorRef"/>; the
/// tests reuse the scripted-actor seam established by the Notification Report
/// page tests (see <c>NotificationReportPageTests</c>).
/// </summary>
public class HealthPageTests : BunitContext
{
private readonly ActorSystem _system = ActorSystem.Create("health-page-tests");
private readonly CommunicationService _comms;
// Mutable scripted reply — individual tests can override before rendering.
private NotificationKpiResponse _kpiReply =
new("k", true, null, QueueDepth: 12, StuckCount: 4, ParkedCount: 3,
DeliveredLastInterval: 88, OldestPendingAge: TimeSpan.FromMinutes(6));
// Site Call Audit (#22) Task 7 — mutable scripted Site Call KPI reply. Tests
// that target the Site Call tiles override this before rendering.
private SiteCallKpiResponse _siteCallKpiReply =
new("k", true, null, BufferedCount: 9, ParkedCount: 2, FailedLastInterval: 1,
DeliveredLastInterval: 40, OldestPendingAge: TimeSpan.FromMinutes(3),
StuckCount: 5);
public HealthPageTests()
{
_comms = new CommunicationService(
Options.Create(new CommunicationOptions()),
NullLogger<CommunicationService>.Instance);
var outbox = _system.ActorOf(Props.Create(() => new ScriptedOutboxActor(this)));
_comms.SetNotificationOutbox(outbox);
var siteCallAudit = _system.ActorOf(Props.Create(() => new ScriptedSiteCallAuditActor(this)));
_comms.SetSiteCallAudit(siteCallAudit);
Services.AddSingleton(_comms);
var aggregator = Substitute.For<ICentralHealthAggregator>();
aggregator.GetAllSiteStates()
.Returns(new Dictionary<string, SiteHealthState>());
Services.AddSingleton(aggregator);
// M6 K16 — the Health page now injects IKpiHistoryQueryService to feed the
// per-site Site Health Trends panel. Stub it with a known non-empty series
// so the page resolves the dependency and the trend charts have data; the
// dedicated trend tests below seed sites / override behaviour.
var kpiHistory = Substitute.For<IKpiHistoryQueryService>();
kpiHistory.GetSeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<KpiSeriesPoint>>(SampleSeries()));
Services.AddSingleton(kpiHistory);
var siteRepo = Substitute.For<ISiteRepository>();
siteRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<Site>>(new List<Site>()));
Services.AddSingleton(siteRepo);
// PLAN-01 Task 14 — the Health page now shows the metrics-stale badge with
// a tooltip naming the configured MetricsStaleTimeout, so it injects the
// health-monitoring options.
Services.AddSingleton(Options.Create(new HealthMonitoringOptions()));
// Audit Log (#23) M7 Bundle E — the Health page now also fetches the
// Audit KPI snapshot. Stub it with an empty point-in-time reading so
// the existing assertions (Notification Outbox tiles, Online/Offline
// counts) keep passing; tests that target the Audit tiles set their
// own substitute.
var auditService = Substitute.For<IAuditLogQueryService>();
auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
.Returns(Task.FromResult(new AuditLogKpiSnapshot(0, 0, 0, DateTime.UtcNow)));
Services.AddSingleton(auditService);
var claims = new[]
{
new Claim(JwtTokenService.UsernameClaimType, "tester"),
new Claim(JwtTokenService.RoleClaimType, "Administrator"),
};
var user = new ClaimsPrincipal(new ClaimsIdentity(claims, "TestAuth"));
Services.AddSingleton<AuthenticationStateProvider>(new TestAuthStateProvider(user));
Services.AddAuthorizationCore();
// The failover control's AuthorizeView evaluates the named RequireAdmin policy, so the
// real policy set has to be registered or the page throws on any card that renders it.
AuthorizationPolicies.AddScadaBridgeAuthorization(Services);
// BunitContext pre-registers a placeholder IAuthorizationService that throws when a
// policy is evaluated; force the real one (same note as NavMenuTests).
Services.AddSingleton<IAuthorizationService, DefaultAuthorizationService>();
// The cluster cards embed CentralFailoverControl (Task 10), which injects these two.
// The page's DI graph genuinely requires them in production, so the test supplies
// them rather than the component making its dependencies optional. Behaviour of the
// control itself is covered by HealthFailoverButtonTests.
Services.AddSingleton(Substitute.For<IManualFailoverService>());
Services.AddSingleton(Substitute.For<IDialogService>());
}
/// <summary>
/// Renders the dashboard the way the app does — inside a
/// <see cref="CascadingAuthenticationState"/>, which the real layout supplies. The
/// cluster cards contain an AuthorizeView (the Task 10 failover control), and
/// AuthorizeView throws without that cascading value.
/// </summary>
private IRenderedComponent<HealthPage> RenderHealthPage()
{
var host = Render<CascadingAuthenticationState>(parameters => parameters
.Add(p => p.ChildContent, (RenderFragment)(builder =>
{
builder.OpenComponent<HealthPage>(0);
builder.CloseComponent();
})));
return host.FindComponent<HealthPage>();
}
[Fact]
public void Renders_OutboxKpiTiles_WithValues()
{
var cut = RenderHealthPage();
// KPI data arrives via an async actor Ask after first render.
cut.WaitForAssertion(() =>
{
Assert.Contains("Notification Outbox", cut.Markup);
Assert.Contains("Queue Depth", cut.Markup);
Assert.Contains("Stuck", cut.Markup);
Assert.Contains("Parked", cut.Markup);
// KPI numeric values surface in the tiles.
Assert.Contains(">12<", cut.Markup); // QueueDepth
Assert.Contains(">4<", cut.Markup); // StuckCount
Assert.Contains(">3<", cut.Markup); // ParkedCount
});
}
[Fact]
public void RendersLinkToTheNotificationKpisPage()
{
var cut = RenderHealthPage();
var link = cut.Find("a[href='/notifications/kpis']");
Assert.Contains("View details", link.TextContent);
}
[Fact]
public void Renders_AuditKpiTiles_WithValues()
{
// Override the default empty snapshot — this test wants concrete values
// to land in the three Audit tiles.
var auditService = Substitute.For<IAuditLogQueryService>();
auditService.GetKpiSnapshotAsync(Arg.Any<CancellationToken>())
.Returns(Task.FromResult(new AuditLogKpiSnapshot(
TotalEventsLastHour: 250,
ErrorEventsLastHour: 5,
BacklogTotal: 17,
AsOfUtc: DateTime.UtcNow)));
Services.AddSingleton(auditService);
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
// The three audit tiles render at the documented data-test selectors.
Assert.Contains("data-test=\"audit-kpi-volume\"", cut.Markup);
Assert.Contains("data-test=\"audit-kpi-error-rate\"", cut.Markup);
Assert.Contains("data-test=\"audit-kpi-backlog\"", cut.Markup);
// Volume shows the formatted thousand-separator value.
Assert.Contains("250", cut.Markup);
// Backlog renders 17.
Assert.Contains("17", cut.Markup);
});
}
[Fact]
public void Renders_SiteCallKpiTiles_WithValues()
{
var cut = RenderHealthPage();
// KPI data arrives via an async actor Ask after first render.
cut.WaitForAssertion(() =>
{
Assert.Contains("Site Calls", cut.Markup);
// The three Site Call tiles render at the documented data-test selectors.
Assert.Contains("data-test=\"site-call-kpi-buffered\"", cut.Markup);
Assert.Contains("data-test=\"site-call-kpi-stuck\"", cut.Markup);
Assert.Contains("data-test=\"site-call-kpi-parked\"", cut.Markup);
// KPI numeric values surface in the tiles.
Assert.Contains(">9<", cut.Markup); // BufferedCount
Assert.Contains(">5<", cut.Markup); // StuckCount
Assert.Contains(">2<", cut.Markup); // ParkedCount
});
}
[Fact]
public void RendersLinkToTheSiteCallsReportPage()
{
var cut = RenderHealthPage();
var link = cut.Find("a[href='/site-calls/report']");
Assert.Contains("View details", link.TextContent);
}
[Fact]
public void SiteCallKpiFailure_ShowsGracefulFallback()
{
_siteCallKpiReply = new SiteCallKpiResponse(
"k", false, "site call repository unavailable", 0, 0, 0, 0, null, 0);
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
// Failure must not crash the page; tiles fall back to a dash and the
// inline error message surfaces.
Assert.Contains("Site Calls", cut.Markup);
Assert.Contains("Site Call KPIs unavailable", cut.Markup);
Assert.Contains("site call repository unavailable", cut.Markup);
Assert.Contains(">—<", cut.Markup);
});
}
[Fact]
public void OutboxKpiFailure_ShowsGracefulFallback()
{
_kpiReply = new NotificationKpiResponse(
"k", false, "outbox repository unavailable", 0, 0, 0, 0, null);
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
// Failure must not crash the page; tiles fall back to a dash.
Assert.Contains("Notification Outbox", cut.Markup);
Assert.Contains("Queue Depth", cut.Markup);
Assert.Contains(">—<", cut.Markup);
});
}
[Fact]
public void Renders_SiteHealthTrends_PanelAndChart_ForSelectedSite()
{
// Seed one site so the trend panel's selector has an option and the
// default-site load produces charts.
SeedSites("site-a");
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
// The panel + its site selector render at the documented hooks.
Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
Assert.Contains("data-test=\"site-health-trends-site\"", cut.Markup);
// The four metric charts render (the shared KpiTrendChart slug hook),
// and the seeded non-empty series draws a polyline. The "S&F Buffer
// Depth" title slugifies to "s-f-buffer-depth" (the & and the spaces
// each collapse to a dash) — see KpiTrendChart.Slugify.
Assert.Contains("kpi-trend-connections-down", cut.Markup);
Assert.Contains("kpi-trend-dead-letters", cut.Markup);
Assert.Contains("kpi-trend-script-errors", cut.Markup);
Assert.Contains("kpi-trend-s-f-buffer-depth", cut.Markup);
Assert.Contains("<polyline", cut.Markup);
});
}
[Fact]
public void SiteHealthTrendsFailure_DoesNotBreakDashboard()
{
SeedSites("site-a");
// The KPI-history service throws on every query — the trend load is
// best-effort, so the dashboard (and its tiles) must still render.
var faulting = Substitute.For<IKpiHistoryQueryService>();
faulting.GetSeriesAsync(
Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<string?>(),
Arg.Any<DateTime>(), Arg.Any<DateTime>(), Arg.Any<int?>(), Arg.Any<CancellationToken>())
.Returns<Task<IReadOnlyList<KpiSeriesPoint>>>(_ =>
throw new InvalidOperationException("kpi history unavailable"));
Services.AddSingleton(faulting);
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
// No unhandled exception: the core dashboard tiles still render, and
// the panel falls back to the per-chart unavailable placeholder.
Assert.Contains("Notification Outbox", cut.Markup);
Assert.Contains("data-test=\"site-health-trends\"", cut.Markup);
Assert.Contains("Trend data unavailable.", cut.Markup);
});
}
[Fact]
public void Renders_MetricsStaleBadge_ForOnlineButStaleSite()
{
// PLAN-01 Task 14: a site that is online (heartbeating) but whose report
// pipeline died shows a distinct "Metrics stale" warning badge.
SeedStates(new SiteHealthState
{
SiteId = "site-a",
IsOnline = true,
IsMetricsStale = true,
LastHeartbeatAt = DateTimeOffset.UtcNow,
LastReportReceivedAt = DateTimeOffset.UtcNow.AddMinutes(-5),
});
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
Assert.Contains("Metrics stale", cut.Markup);
// The badge uses the page's Bootstrap warning style.
Assert.Contains("bg-warning", cut.Markup);
});
}
[Fact]
public void Renders_OfflineSince_ForOfflineSiteWithStatusChange()
{
var changedAt = new DateTimeOffset(2026, 7, 8, 12, 34, 56, TimeSpan.Zero);
SeedStates(new SiteHealthState
{
SiteId = "site-a",
IsOnline = false,
LastHeartbeatAt = changedAt,
LastStatusChangeAt = changedAt,
});
var cut = RenderHealthPage();
cut.WaitForAssertion(() =>
{
Assert.Contains("offline since", cut.Markup);
// The "u" (universal sortable) format the page renders the flip with.
Assert.Contains(changedAt.ToString("u"), cut.Markup);
});
}
// Re-seeds the aggregator substitute with fully-specified states (as opposed
// to SeedSites' minimal online placeholders).
private void SeedStates(params SiteHealthState[] states)
{
var aggregator = Substitute.For<ICentralHealthAggregator>();
aggregator.GetAllSiteStates()
.Returns(states.ToDictionary(s => s.SiteId));
Services.AddSingleton(aggregator);
}
// Re-seeds the aggregator substitute so the trend panel's site selector has
// options. Each site id maps to a minimal online SiteHealthState (a null
// report is fine — the trend panel keys off the site ids, not the report).
private void SeedSites(params string[] siteIds)
{
var aggregator = Substitute.For<ICentralHealthAggregator>();
var states = siteIds.ToDictionary(
id => id,
id => new SiteHealthState
{
SiteId = id,
IsOnline = true,
LastHeartbeatAt = DateTimeOffset.UtcNow,
});
aggregator.GetAllSiteStates()
.Returns(new Dictionary<string, SiteHealthState>(states));
Services.AddSingleton(aggregator);
}
// A known non-empty (≥2-point) series so KpiTrendChart renders a polyline
// rather than the single-sample / unavailable placeholder.
private static IReadOnlyList<KpiSeriesPoint> SampleSeries()
{
var baseUtc = DateTime.UtcNow.AddHours(-24);
return new List<KpiSeriesPoint>
{
new(baseUtc, 1),
new(baseUtc.AddHours(6), 3),
new(baseUtc.AddHours(12), 2),
new(baseUtc.AddHours(18), 5),
};
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_system.Terminate().Wait(TimeSpan.FromSeconds(5));
}
base.Dispose(disposing);
}
/// <summary>
/// Stand-in for the notification-outbox actor. Replies to the KPI request
/// with the test's currently-scripted response.
/// </summary>
private sealed class ScriptedOutboxActor : ReceiveActor
{
public ScriptedOutboxActor(HealthPageTests test)
{
Receive<NotificationKpiRequest>(_ => Sender.Tell(test._kpiReply));
}
}
/// <summary>
/// Stand-in for the Site Call Audit actor. Replies to the KPI request with
/// the test's currently-scripted response. Also handles the per-node KPI
/// request (T6: M5.2) with an empty-nodes success reply so the Health page
/// can complete initialization without a 30-second Ask timeout.
/// </summary>
private sealed class ScriptedSiteCallAuditActor : ReceiveActor
{
public ScriptedSiteCallAuditActor(HealthPageTests test)
{
Receive<SiteCallKpiRequest>(_ => Sender.Tell(test._siteCallKpiReply));
Receive<PerNodeSiteCallKpiRequest>(req => Sender.Tell(
new PerNodeSiteCallKpiResponse(req.CorrelationId, Success: true, ErrorMessage: null,
Nodes: Array.Empty<SiteCallNodeKpiSnapshot>())));
}
}
}