feat(overview): scaffold the family dashboard + registry + health client
Phase 3 tasks 3.1-3.3 of docs/plans/2026-07-22-overview-dashboard-impl-plan.md. Scaffold (3.1): a plain directory in scadaproj (NOT a nested git repo), slnx over src/ + tests/, HistorianGateway-pattern Directory.Build.props (warnings-as-errors) and nuget.config (nuget.org * + the Gitea feed scoped to ZB.MOM.WW.*). Inline package pins, no CPM — the app convention here. NO Auth/Audit/Secrets packages: the dashboard is anonymous and read-only by requirement. Registry (3.2): OverviewOptions/ApplicationEntry/InstanceEntry bound from the "Overview" section, plus EffectiveTimings resolving instance > application > global overrides in ONE place so no caller re-implements the precedence. OverviewOptionsValidator (OptionsValidatorBase + AddValidatedOptions) rejects an empty registry, applications with no instances, duplicate names, non-absolute or non-http(s) URLs, and non-positive intervals — and checks stale > poll on the EFFECTIVE values, since an override at either level can invert that on one instance while the globals look fine. A ConfigPreflight presence check runs before the host is built so a missing registry fails with the key name, not with "Applications must have at least 1 entry". Health client (3.3): ZbHealthReport DTOs for the canonical ZbHealthWriter body, with per-entry `data` kept as JsonElement — it is an open contract, and binding it to concrete types would break the moment a check adds a field. 200 AND 503 are both parseable answers (503 carries the body naming the failing check); only a transport failure or a non-canonical body is a failed probe, which is why Unreachable stays distinct from Down. Host shutdown propagates as cancellation rather than being recorded as every instance timing out. Status model: Up/Degraded/Down/Unreachable + Active/Standby/Unknown/NotApplicable, with two-strike flap damping — a failing observation only replaces a good state after 2 consecutive failures, while recovery is immediate (damping suppresses false alarms; symmetric damping would just make them linger). 53 tests green, 0 warnings in Debug and Release. Tests for the validator, the client golden payloads (with AND without `data`, so a partly-bumped fleet renders) and the derivation table are front-loaded here from task 3.8 so this batch is verified rather than pending. Also pins AngleSharp 1.5.2 test-side: bunit 1.40.0 pulls 1.2.0, which trips NU1902 and therefore the warnings-as-errors gate (same fix HistorianGateway made in 6bc005d).
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,32 @@
|
||||
<Project>
|
||||
|
||||
<!--
|
||||
Workspace-wide build settings for ZB.MOM.WW.Overview (the family overview dashboard).
|
||||
Mirrors the HistorianGateway workspace props — the reference app layout for this family.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<LangVersion>latest</LangVersion>
|
||||
</PropertyGroup>
|
||||
|
||||
<!--
|
||||
Build-quality floor: the family's zero-warnings invariant is a build GATE, not a convention —
|
||||
new warnings break the build and are fixed rather than suppressed.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<AnalysisLevel>latest</AnalysisLevel>
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
<Deterministic>true</Deterministic>
|
||||
</PropertyGroup>
|
||||
|
||||
<!--
|
||||
This is an APP, not a library: nothing here is ever packed to the Gitea feed.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,8 @@
|
||||
<Solution>
|
||||
<Folder Name="/src/">
|
||||
<Project Path="src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/tests/">
|
||||
<Project Path="tests/ZB.MOM.WW.Overview.Tests/ZB.MOM.WW.Overview.Tests.csproj" />
|
||||
</Folder>
|
||||
</Solution>
|
||||
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<packageSources>
|
||||
<clear />
|
||||
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
|
||||
<add key="dohertj2-gitea" value="https://gitea.dohertylan.com/api/packages/dohertj2/nuget/index.json" />
|
||||
</packageSources>
|
||||
<!-- nuget.org serves everything; the Gitea feed serves only the ZB.MOM.WW.* shared libs.
|
||||
Credentials are NOT committed: they are provided per-developer at the user level
|
||||
(~/.nuget/NuGet/NuGet.Config). -->
|
||||
<packageSourceMapping>
|
||||
<packageSource key="nuget.org">
|
||||
<package pattern="*" />
|
||||
</packageSource>
|
||||
<packageSource key="dohertj2-gitea">
|
||||
<package pattern="ZB.MOM.WW.Health" />
|
||||
<package pattern="ZB.MOM.WW.Health.*" />
|
||||
<package pattern="ZB.MOM.WW.Telemetry" />
|
||||
<package pattern="ZB.MOM.WW.Telemetry.*" />
|
||||
<package pattern="ZB.MOM.WW.Configuration" />
|
||||
<package pattern="ZB.MOM.WW.Theme" />
|
||||
</packageSource>
|
||||
</packageSourceMapping>
|
||||
</configuration>
|
||||
@@ -0,0 +1,132 @@
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>The derived state of one probed instance.</summary>
|
||||
public enum InstanceStatus
|
||||
{
|
||||
/// <summary>Ready tier answered 200 / <c>Healthy</c>.</summary>
|
||||
Up,
|
||||
|
||||
/// <summary>Ready tier answered 200 / <c>Degraded</c> — serving, but a check is unhappy.</summary>
|
||||
Degraded,
|
||||
|
||||
/// <summary>
|
||||
/// Ready tier answered 503 / <c>Unhealthy</c>. The app is up and talking; a dependency is not.
|
||||
/// </summary>
|
||||
Down,
|
||||
|
||||
/// <summary>
|
||||
/// Nothing answered — timeout, connection refused, DNS failure, or a body that is not the
|
||||
/// canonical health JSON. Kept distinct from <see cref="Down"/> because it usually means
|
||||
/// network / VPN / registry typo rather than a sick application, and the two lead an operator
|
||||
/// to look in completely different places.
|
||||
/// </summary>
|
||||
Unreachable,
|
||||
}
|
||||
|
||||
/// <summary>Whether a probed instance is the active half of its pair.</summary>
|
||||
public enum ActiveState
|
||||
{
|
||||
/// <summary>The instance has no active/standby concept (single-instance gateways).</summary>
|
||||
NotApplicable,
|
||||
|
||||
/// <summary>The active probe did not answer, so activity is unknown.</summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>Active tier answered 200 — this is the active node.</summary>
|
||||
Active,
|
||||
|
||||
/// <summary>Active tier answered 503 — this is the standby.</summary>
|
||||
Standby,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pure status derivation: probe result → <see cref="InstanceStatus"/>, with two-strike flap
|
||||
/// damping applied across polls. Static and dependency-free so every rule is table-testable.
|
||||
/// </summary>
|
||||
public static class StatusDerivation
|
||||
{
|
||||
/// <summary>
|
||||
/// Consecutive failing observations required before a previously-good instance is shown as
|
||||
/// failing. A single blip — one dropped packet, one GC pause past the timeout — is not news;
|
||||
/// two in a row is.
|
||||
/// </summary>
|
||||
public const int FailureStrikes = 2;
|
||||
|
||||
/// <summary>Classifies one ready-tier probe, before any damping.</summary>
|
||||
/// <param name="probe">The probe result.</param>
|
||||
/// <returns>The observed status.</returns>
|
||||
public static InstanceStatus Observe(HealthProbeResult probe)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(probe);
|
||||
|
||||
if (!probe.Reachable || probe.Report is null)
|
||||
return InstanceStatus.Unreachable;
|
||||
|
||||
// The reported status wins over the HTTP code: they agree by construction (Healthy/Degraded
|
||||
// → 200, Unhealthy → 503), and the body is the more specific of the two — 200 alone cannot
|
||||
// tell Up from Degraded.
|
||||
return probe.Report.Status switch
|
||||
{
|
||||
"Healthy" => InstanceStatus.Up,
|
||||
"Degraded" => InstanceStatus.Degraded,
|
||||
"Unhealthy" => InstanceStatus.Down,
|
||||
|
||||
// Parsed as JSON but carrying no status we recognise: the endpoint is answering with
|
||||
// something that is not this contract, which is the Unreachable case, not a sick app.
|
||||
_ => InstanceStatus.Unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Classifies one active-tier probe.</summary>
|
||||
/// <param name="hasActiveRole">Whether this instance participates in an active/standby pair.</param>
|
||||
/// <param name="probe">The active-tier probe result, or null when not probed.</param>
|
||||
/// <returns>The derived active state.</returns>
|
||||
public static ActiveState ObserveActive(bool hasActiveRole, HealthProbeResult? probe)
|
||||
{
|
||||
if (!hasActiveRole)
|
||||
return ActiveState.NotApplicable;
|
||||
|
||||
if (probe is null || !probe.Reachable || probe.StatusCode is null)
|
||||
return ActiveState.Unknown;
|
||||
|
||||
return probe.StatusCode switch
|
||||
{
|
||||
200 => ActiveState.Active,
|
||||
503 => ActiveState.Standby,
|
||||
_ => ActiveState.Unknown,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies two-strike damping: a failing observation only replaces a previously-good state
|
||||
/// after <see cref="FailureStrikes"/> consecutive failures, while any good observation is
|
||||
/// adopted immediately.
|
||||
/// </summary>
|
||||
/// <param name="previous">The last displayed status, or null on the first-ever poll.</param>
|
||||
/// <param name="previousConsecutiveFailures">Consecutive failures recorded before this poll.</param>
|
||||
/// <param name="observed">This poll's observed status.</param>
|
||||
/// <returns>The status to display and the updated consecutive-failure count.</returns>
|
||||
/// <remarks>
|
||||
/// Recovery is deliberately asymmetric — instant. Damping exists to suppress false alarms, and
|
||||
/// making recovery equally slow would just convert them into false alarms that linger.
|
||||
/// </remarks>
|
||||
public static (InstanceStatus Status, int ConsecutiveFailures) Damp(
|
||||
InstanceStatus? previous,
|
||||
int previousConsecutiveFailures,
|
||||
InstanceStatus observed)
|
||||
{
|
||||
var failing = observed is InstanceStatus.Down or InstanceStatus.Unreachable;
|
||||
|
||||
if (!failing)
|
||||
return (observed, 0);
|
||||
|
||||
var failures = previousConsecutiveFailures + 1;
|
||||
|
||||
// With no previous state there is nothing to protect and nothing better to show: adopt the
|
||||
// observation. Claiming Up on a first poll that failed would be a fabrication.
|
||||
if (previous is null || failures >= FailureStrikes)
|
||||
return (observed, failures);
|
||||
|
||||
return (previous.Value, failures);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// The canonical <c>ZbHealthWriter</c> body every family app serves from <c>MapZbHealth</c>:
|
||||
/// <c>{ status, totalDurationMs, entries: { name: { status, description, durationMs, data? } } }</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This mirrors a contract owned by <c>ZB.MOM.WW.Health</c>. It lives in the app for v1 by
|
||||
/// deliberate decision (design §8.2): promoting it to a <c>ZB.MOM.WW.Health.Client</c> package is
|
||||
/// worth a publish only once a second consumer exists.
|
||||
/// </remarks>
|
||||
public sealed class ZbHealthReport
|
||||
{
|
||||
/// <summary>Aggregate tier status: <c>Healthy</c>, <c>Degraded</c> or <c>Unhealthy</c>.</summary>
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; init; }
|
||||
|
||||
/// <summary>Total time the tier's checks took, in milliseconds.</summary>
|
||||
[JsonPropertyName("totalDurationMs")]
|
||||
public double TotalDurationMs { get; init; }
|
||||
|
||||
/// <summary>Per-check results, keyed by check name (verbatim, e.g. <c>akka-cluster</c>).</summary>
|
||||
[JsonPropertyName("entries")]
|
||||
public Dictionary<string, ZbHealthEntry> Entries { get; init; } = new(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>One check's result inside a <see cref="ZbHealthReport"/>.</summary>
|
||||
public sealed class ZbHealthEntry
|
||||
{
|
||||
/// <summary>This check's status: <c>Healthy</c>, <c>Degraded</c> or <c>Unhealthy</c>.</summary>
|
||||
[JsonPropertyName("status")]
|
||||
public string? Status { get; init; }
|
||||
|
||||
/// <summary>Human-readable detail. Always present in the canonical shape; may be JSON null.</summary>
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>How long this check took, in milliseconds.</summary>
|
||||
[JsonPropertyName("durationMs")]
|
||||
public double DurationMs { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional structured data the check published (ZB.MOM.WW.Health 0.2.0+). Absent entirely on
|
||||
/// checks that publish none and on every pre-0.2.0 node — hence nullable, and hence the
|
||||
/// dashboard must render fully (minus leader) without it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Values stay as <see cref="JsonElement"/>: this is an open contract carrying arbitrary JSON,
|
||||
/// and binding it to concrete types here would break the moment a check adds a field.
|
||||
/// </remarks>
|
||||
[JsonPropertyName("data")]
|
||||
public Dictionary<string, JsonElement>? Data { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Reads <paramref name="key"/> from <see cref="Data"/> as a string, or null when the key is
|
||||
/// absent or is not a JSON string.
|
||||
/// </summary>
|
||||
/// <param name="key">The data key, matched verbatim.</param>
|
||||
/// <returns>The string value, or null.</returns>
|
||||
public string? DataString(string key)
|
||||
{
|
||||
if (Data is null || !Data.TryGetValue(key, out var value))
|
||||
return null;
|
||||
|
||||
return value.ValueKind == JsonValueKind.String ? value.GetString() : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads <paramref name="key"/> from <see cref="Data"/> as an integer, or null when the key is
|
||||
/// absent or is not a JSON number.
|
||||
/// </summary>
|
||||
/// <param name="key">The data key, matched verbatim.</param>
|
||||
/// <returns>The integer value, or null.</returns>
|
||||
public int? DataInt(string key)
|
||||
{
|
||||
if (Data is null || !Data.TryGetValue(key, out var value))
|
||||
return null;
|
||||
|
||||
return value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var parsed) ? parsed : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
/// <summary>
|
||||
/// One probe of one health endpoint: whether anything answered, what it answered, and how long it
|
||||
/// took.
|
||||
/// </summary>
|
||||
/// <param name="Reachable">
|
||||
/// True when the endpoint answered with a parseable canonical body. False for timeouts, refused
|
||||
/// connections, DNS failures, and bodies that are not the canonical health JSON.
|
||||
/// </param>
|
||||
/// <param name="StatusCode">The HTTP status code, when one was received.</param>
|
||||
/// <param name="Report">The parsed body, when it parsed.</param>
|
||||
/// <param name="Latency">Wall-clock time for the request.</param>
|
||||
/// <param name="Error">A short reason when <paramref name="Reachable"/> is false.</param>
|
||||
public sealed record HealthProbeResult(
|
||||
bool Reachable,
|
||||
int? StatusCode,
|
||||
ZbHealthReport? Report,
|
||||
TimeSpan Latency,
|
||||
string? Error);
|
||||
|
||||
/// <summary>
|
||||
/// GETs a family health endpoint and parses the canonical <c>ZbHealthWriter</c> body.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Both 200 and 503 are treated as SUCCESSFUL probes: an Unhealthy tier still returns a full body,
|
||||
/// and that body is exactly what the dashboard needs to show which check failed. Only a genuine
|
||||
/// non-answer (transport failure) or an unparseable body is a failed probe.
|
||||
/// </remarks>
|
||||
public sealed class ZbHealthReportClient
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
/// <summary>Initializes a new <see cref="ZbHealthReportClient"/>.</summary>
|
||||
/// <param name="httpClient">The HTTP client used for probes.</param>
|
||||
public ZbHealthReportClient(HttpClient httpClient) =>
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
|
||||
/// <summary>Probes <paramref name="url"/> and parses the response.</summary>
|
||||
/// <param name="url">Absolute URL of a health endpoint.</param>
|
||||
/// <param name="timeout">Per-request timeout.</param>
|
||||
/// <param name="cancellationToken">Token cancelled when the host is shutting down.</param>
|
||||
/// <returns>The probe result; this never throws for an unreachable endpoint.</returns>
|
||||
public async Task<HealthProbeResult> ProbeAsync(
|
||||
string url,
|
||||
TimeSpan timeout,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var startedAt = Stopwatch.GetTimestamp();
|
||||
|
||||
// Linked, not a bare timeout token: shutdown must cancel an in-flight probe immediately,
|
||||
// and the two causes have to stay distinguishable so a shutdown is not misreported as a
|
||||
// timed-out instance.
|
||||
using var timeoutSource = new CancellationTokenSource(timeout);
|
||||
using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token);
|
||||
|
||||
try
|
||||
{
|
||||
using var response = await _httpClient
|
||||
.GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false);
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
ZbHealthReport? report;
|
||||
try
|
||||
{
|
||||
report = JsonSerializer.Deserialize<ZbHealthReport>(body, SerializerOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
// Answered, but not with this contract — a reverse proxy error page, an HTML login
|
||||
// redirect, a wrong port. That is a registry/plumbing problem, not a sick app, so
|
||||
// it lands as Unreachable rather than Down.
|
||||
return Failed(startedAt, statusCode, $"response body is not canonical health JSON: {ex.Message}");
|
||||
}
|
||||
|
||||
if (report is null)
|
||||
return Failed(startedAt, statusCode, "response body deserialized to null");
|
||||
|
||||
return new HealthProbeResult(true, statusCode, report, Stopwatch.GetElapsedTime(startedAt), null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Host shutdown, not an instance fault — let the caller abandon the sweep.
|
||||
throw;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return Failed(startedAt, null, $"timed out after {timeout.TotalSeconds:0.##}s");
|
||||
}
|
||||
catch (HttpRequestException ex)
|
||||
{
|
||||
return Failed(startedAt, null, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) =>
|
||||
new(false, statusCode, null, Stopwatch.GetElapsedTime(startedAt), error);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
// Pre-host presence check. The validator below covers shape and consistency, but it only runs once
|
||||
// the options are resolved — a registry section that is missing entirely binds to an empty
|
||||
// OverviewOptions, and the failure a reader can act on is "you configured no registry", not
|
||||
// "Applications must have at least 1 entry". Fail here, before the host is built, with the key name.
|
||||
ConfigPreflight.For(builder.Configuration)
|
||||
.RequireValue($"{OverviewOptions.SectionName}:Applications:0:Name")
|
||||
.RequireValue($"{OverviewOptions.SectionName}:Applications:0:Instances:0:BaseUrl")
|
||||
.ThrowIfInvalid();
|
||||
|
||||
builder.Services.AddValidatedOptions<OverviewOptions, OverviewOptionsValidator>(
|
||||
builder.Configuration, OverviewOptions.SectionName);
|
||||
|
||||
var app = builder.Build();
|
||||
|
||||
app.MapGet("/", () => "ZB.MOM.WW.Overview");
|
||||
|
||||
await app.RunAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Exposed so <c>WebApplicationFactory<Program></c> can boot the real host in tests
|
||||
/// (top-level statements otherwise compile to an internal entry-point class).
|
||||
/// </summary>
|
||||
public partial class Program;
|
||||
@@ -0,0 +1,124 @@
|
||||
namespace ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// The dashboard's registry: which application instances to probe, and how often. Bound from the
|
||||
/// <c>Overview</c> configuration section and validated by <see cref="OverviewOptionsValidator"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every instance is probed the same way — GET <c>{BaseUrl}/health/ready</c>, plus
|
||||
/// <c>{BaseUrl}/health/active</c> when <see cref="InstanceEntry.HasActiveRole"/>. There is
|
||||
/// deliberately no per-instance probe-mode switch: one probe model, one status derivation.
|
||||
/// </remarks>
|
||||
public sealed class OverviewOptions
|
||||
{
|
||||
/// <summary>The configuration section this binds to.</summary>
|
||||
public const string SectionName = "Overview";
|
||||
|
||||
/// <summary>How often the poller sweeps every instance, in seconds.</summary>
|
||||
public int PollIntervalSeconds { get; set; } = 10;
|
||||
|
||||
/// <summary>Per-request timeout, in seconds.</summary>
|
||||
public int TimeoutSeconds { get; set; } = 3;
|
||||
|
||||
/// <summary>
|
||||
/// How old a snapshot may get before its instance renders as Stale (out of date / offline)
|
||||
/// regardless of its last known status, in seconds. This is what catches a stuck poller, a
|
||||
/// paused refresh or a slept machine — not just failed probes — so it must exceed the poll
|
||||
/// interval or every instance would read Stale between two healthy polls.
|
||||
/// </summary>
|
||||
public int StaleAfterSeconds { get; set; } = 45;
|
||||
|
||||
/// <summary>The registered applications, each with one or more instances.</summary>
|
||||
public IList<ApplicationEntry> Applications { get; set; } = new List<ApplicationEntry>();
|
||||
}
|
||||
|
||||
/// <summary>One registered application (a section on the dashboard).</summary>
|
||||
public sealed class ApplicationEntry
|
||||
{
|
||||
/// <summary>Display name, unique across the registry.</summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Label for the per-instance management link, e.g. "AdminUI" or "Dashboard".</summary>
|
||||
public string ManagementLabel { get; set; } = "UI";
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.PollIntervalSeconds"/> for this application.</summary>
|
||||
public int? PollIntervalSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.TimeoutSeconds"/> for this application.</summary>
|
||||
public int? TimeoutSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides <see cref="OverviewOptions.StaleAfterSeconds"/> for this application.</summary>
|
||||
public int? StaleAfterSeconds { get; set; }
|
||||
|
||||
/// <summary>This application's instances; at least one is required.</summary>
|
||||
public IList<InstanceEntry> Instances { get; set; } = new List<InstanceEntry>();
|
||||
}
|
||||
|
||||
/// <summary>One probed instance (a card on the dashboard).</summary>
|
||||
public sealed class InstanceEntry
|
||||
{
|
||||
/// <summary>Display name, unique within its application.</summary>
|
||||
public string Name { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The cluster group this instance belongs to (e.g. "central", "site-a"). One Akka cluster is
|
||||
/// one group: leader aggregation and the split-brain check are scoped to it. Null for
|
||||
/// standalone instances.
|
||||
/// </summary>
|
||||
public string? Group { get; set; }
|
||||
|
||||
/// <summary>Absolute base URL whose <c>/health/*</c> endpoints are probed.</summary>
|
||||
public string BaseUrl { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Absolute URL of this instance's own management UI, linked from its card.</summary>
|
||||
public string? ManagementUrl { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this instance participates in an active/standby pair. False for the single-instance
|
||||
/// gateways, which have no such concept — their cards show status without an Active badge and
|
||||
/// are never probed for <c>/health/active</c>.
|
||||
/// </summary>
|
||||
public bool HasActiveRole { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global poll interval for this instance.</summary>
|
||||
public int? PollIntervalSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global request timeout for this instance.</summary>
|
||||
public int? TimeoutSeconds { get; set; }
|
||||
|
||||
/// <summary>Overrides the application and global staleness window for this instance.</summary>
|
||||
public int? StaleAfterSeconds { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The effective (override-resolved) timings for one instance. Instance wins over application,
|
||||
/// application wins over global — resolved once here so no caller re-implements the precedence.
|
||||
/// </summary>
|
||||
/// <param name="PollInterval">Effective poll cadence.</param>
|
||||
/// <param name="Timeout">Effective per-request timeout.</param>
|
||||
/// <param name="StaleAfter">Effective staleness window.</param>
|
||||
public readonly record struct EffectiveTimings(TimeSpan PollInterval, TimeSpan Timeout, TimeSpan StaleAfter)
|
||||
{
|
||||
/// <summary>Resolves the effective timings for <paramref name="instance"/>.</summary>
|
||||
/// <param name="options">The global registry defaults.</param>
|
||||
/// <param name="application">The instance's application.</param>
|
||||
/// <param name="instance">The instance itself.</param>
|
||||
/// <returns>The resolved timings.</returns>
|
||||
public static EffectiveTimings Resolve(
|
||||
OverviewOptions options,
|
||||
ApplicationEntry application,
|
||||
InstanceEntry instance)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(application);
|
||||
ArgumentNullException.ThrowIfNull(instance);
|
||||
|
||||
return new EffectiveTimings(
|
||||
TimeSpan.FromSeconds(
|
||||
instance.PollIntervalSeconds ?? application.PollIntervalSeconds ?? options.PollIntervalSeconds),
|
||||
TimeSpan.FromSeconds(
|
||||
instance.TimeoutSeconds ?? application.TimeoutSeconds ?? options.TimeoutSeconds),
|
||||
TimeSpan.FromSeconds(
|
||||
instance.StaleAfterSeconds ?? application.StaleAfterSeconds ?? options.StaleAfterSeconds));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
/// <summary>
|
||||
/// Validates the <c>Overview</c> registry. Registered through
|
||||
/// <c>AddValidatedOptions<OverviewOptions, OverviewOptionsValidator></c>, so a malformed
|
||||
/// registry fails the host at startup rather than surfacing as a silently-empty dashboard or a
|
||||
/// first-poll exception.
|
||||
/// </summary>
|
||||
public sealed class OverviewOptionsValidator : OptionsValidatorBase<OverviewOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, OverviewOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(builder);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
RequirePositive(builder, options.PollIntervalSeconds, "Overview:PollIntervalSeconds");
|
||||
RequirePositive(builder, options.TimeoutSeconds, "Overview:TimeoutSeconds");
|
||||
RequirePositive(builder, options.StaleAfterSeconds, "Overview:StaleAfterSeconds");
|
||||
|
||||
builder.MinCount(options.Applications as IReadOnlyCollection<ApplicationEntry>, 1, "Overview:Applications");
|
||||
|
||||
var seenApplications = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var a = 0; a < options.Applications.Count; a++)
|
||||
{
|
||||
var application = options.Applications[a];
|
||||
var appPath = $"Overview:Applications:{a}";
|
||||
|
||||
builder.Required(application.Name, $"{appPath}:Name");
|
||||
if (!string.IsNullOrWhiteSpace(application.Name) && !seenApplications.Add(application.Name))
|
||||
builder.Add($"{appPath}:Name '{application.Name}' is duplicated; application names must be unique");
|
||||
|
||||
RequirePositiveIfSet(builder, application.PollIntervalSeconds, $"{appPath}:PollIntervalSeconds");
|
||||
RequirePositiveIfSet(builder, application.TimeoutSeconds, $"{appPath}:TimeoutSeconds");
|
||||
RequirePositiveIfSet(builder, application.StaleAfterSeconds, $"{appPath}:StaleAfterSeconds");
|
||||
|
||||
builder.MinCount(
|
||||
application.Instances as IReadOnlyCollection<InstanceEntry>, 1, $"{appPath}:Instances");
|
||||
|
||||
var seenInstances = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var i = 0; i < application.Instances.Count; i++)
|
||||
{
|
||||
var instance = application.Instances[i];
|
||||
var path = $"{appPath}:Instances:{i}";
|
||||
|
||||
builder.Required(instance.Name, $"{path}:Name");
|
||||
if (!string.IsNullOrWhiteSpace(instance.Name) && !seenInstances.Add(instance.Name))
|
||||
{
|
||||
builder.Add(
|
||||
$"{path}:Name '{instance.Name}' is duplicated within application "
|
||||
+ $"'{application.Name}'; instance names must be unique per application");
|
||||
}
|
||||
|
||||
RequireAbsoluteHttpUrl(builder, instance.BaseUrl, $"{path}:BaseUrl", required: true);
|
||||
RequireAbsoluteHttpUrl(builder, instance.ManagementUrl, $"{path}:ManagementUrl", required: false);
|
||||
|
||||
RequirePositiveIfSet(builder, instance.PollIntervalSeconds, $"{path}:PollIntervalSeconds");
|
||||
RequirePositiveIfSet(builder, instance.TimeoutSeconds, $"{path}:TimeoutSeconds");
|
||||
RequirePositiveIfSet(builder, instance.StaleAfterSeconds, $"{path}:StaleAfterSeconds");
|
||||
|
||||
// Checked on the EFFECTIVE values, not the globals: an override at either level can
|
||||
// invert the relationship on one instance while the global pair stays sane, and an
|
||||
// instance whose stale window is inside its poll interval would render Stale
|
||||
// between two perfectly healthy polls.
|
||||
var timings = EffectiveTimings.Resolve(options, application, instance);
|
||||
if (timings.StaleAfter <= timings.PollInterval)
|
||||
{
|
||||
builder.Add(
|
||||
$"{path}: effective StaleAfterSeconds ({timings.StaleAfter.TotalSeconds:0.##}) must be "
|
||||
+ $"greater than effective PollIntervalSeconds ({timings.PollInterval.TotalSeconds:0.##}) "
|
||||
+ "— otherwise this instance renders Stale between two healthy polls");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void RequirePositive(ValidationBuilder builder, int value, string field) =>
|
||||
builder.RequireThat(value > 0, $"{field} must be greater than 0");
|
||||
|
||||
private static void RequirePositiveIfSet(ValidationBuilder builder, int? value, string field)
|
||||
{
|
||||
if (value is { } set)
|
||||
RequirePositive(builder, set, field);
|
||||
}
|
||||
|
||||
private static void RequireAbsoluteHttpUrl(
|
||||
ValidationBuilder builder, string? value, string field, bool required)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
if (required)
|
||||
builder.Add($"{field} is required");
|
||||
return;
|
||||
}
|
||||
|
||||
// Absolute + http(s) explicitly: a relative URL binds without complaint and only fails at
|
||||
// the first poll, as an opaque HttpClient error against a dashboard that has no base
|
||||
// address of its own.
|
||||
if (!Uri.TryCreate(value, UriKind.Absolute, out var uri)
|
||||
|| (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps))
|
||||
{
|
||||
builder.Add($"{field} must be an absolute http:// or https:// URL (was '{value}')");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk.Web">
|
||||
|
||||
<!--
|
||||
The family overview dashboard: a Blazor Server app that polls every registered instance's
|
||||
anonymous /health/ready + /health/active endpoints and renders one cached page.
|
||||
|
||||
Inline pinned versions, deliberately NO central package management: that is the app
|
||||
convention in this family (mxgw and HistorianGateway carry inline pins too, and a
|
||||
CPM-shaped sweep silently reports such repos as empty).
|
||||
|
||||
NO Auth / Audit / Secrets packages. The dashboard is anonymous and read-only by
|
||||
requirement — there is no login, no mutating action to audit, and nothing secret in the
|
||||
registry. That is a requirement, not an omission; do not add them "while at it".
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<RootNamespace>ZB.MOM.WW.Overview</RootNamespace>
|
||||
<UserSecretsId>zb-mom-ww-overview</UserSecretsId>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Exposes the polling internals (snapshot store, status derivation, leader resolution)
|
||||
to the test project so they are testable without going through the UI. -->
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.Overview.Tests" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ZB.MOM.WW.Theme" Version="0.3.1" />
|
||||
<PackageReference Include="ZB.MOM.WW.Health" Version="0.2.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry.Serilog" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Configuration" Version="0.1.0" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
|
||||
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,226 @@
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.Overview.Registry;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Registry validation. A malformed registry must fail the host at startup with a message naming
|
||||
/// the offending key — the alternative is a dashboard that boots and quietly shows nothing, or one
|
||||
/// that throws on its first poll where nobody is looking.
|
||||
/// </summary>
|
||||
public class OverviewOptionsValidatorTests
|
||||
{
|
||||
private static readonly OverviewOptionsValidator Validator = new();
|
||||
|
||||
private static OverviewOptions Valid(Action<OverviewOptions>? mutate = null)
|
||||
{
|
||||
var options = new OverviewOptions
|
||||
{
|
||||
PollIntervalSeconds = 10,
|
||||
TimeoutSeconds = 3,
|
||||
StaleAfterSeconds = 45,
|
||||
Applications =
|
||||
{
|
||||
new ApplicationEntry
|
||||
{
|
||||
Name = "OtOpcUa",
|
||||
ManagementLabel = "AdminUI",
|
||||
Instances =
|
||||
{
|
||||
new InstanceEntry
|
||||
{
|
||||
Name = "central-1",
|
||||
Group = "central",
|
||||
BaseUrl = "http://otopcua-central-1:9000",
|
||||
ManagementUrl = "http://otopcua-central-1:9000/",
|
||||
HasActiveRole = true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
mutate?.Invoke(options);
|
||||
return options;
|
||||
}
|
||||
|
||||
private static ValidateOptionsResult Run(OverviewOptions options) => Validator.Validate(null, options);
|
||||
|
||||
private static void AssertFailsWith(OverviewOptions options, string expectedFragment)
|
||||
{
|
||||
var result = Run(options);
|
||||
|
||||
Assert.True(result.Failed, "expected validation to fail");
|
||||
Assert.Contains(
|
||||
result.Failures,
|
||||
f => f.Contains(expectedFragment, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidRegistry_Succeeds()
|
||||
{
|
||||
// Positive control: without this, every "fails with X" assertion below could be passing
|
||||
// because the fixture itself is malformed for some unrelated reason.
|
||||
Assert.True(Run(Valid()).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EmptyRegistry_IsRejected()
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.Applications.Clear()), "Overview:Applications");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplicationWithNoInstances_IsRejected()
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.Applications[0].Instances.Clear()), "Instances");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
public void ApplicationWithoutName_IsRejected(string name)
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.Applications[0].Name = name), "Applications:0:Name");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DuplicateApplicationNames_AreRejected()
|
||||
{
|
||||
AssertFailsWith(
|
||||
Valid(o => o.Applications.Add(new ApplicationEntry
|
||||
{
|
||||
Name = "OtOpcUa",
|
||||
Instances = { new InstanceEntry { Name = "other", BaseUrl = "http://other:9000" } },
|
||||
})),
|
||||
"duplicated");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DuplicateInstanceNamesWithinAnApplication_AreRejected()
|
||||
{
|
||||
AssertFailsWith(
|
||||
Valid(o => o.Applications[0].Instances.Add(new InstanceEntry
|
||||
{
|
||||
Name = "central-1",
|
||||
BaseUrl = "http://otopcua-central-2:9000",
|
||||
})),
|
||||
"duplicated");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SameInstanceNameInDifferentApplications_IsAllowed()
|
||||
{
|
||||
// "central-1" under OtOpcUa and "central-1" under ScadaBridge are different machines with
|
||||
// the same conventional name — uniqueness is per application, not global.
|
||||
var options = Valid(o => o.Applications.Add(new ApplicationEntry
|
||||
{
|
||||
Name = "ScadaBridge",
|
||||
Instances = { new InstanceEntry { Name = "central-1", BaseUrl = "http://sb-central-a:5000" } },
|
||||
}));
|
||||
|
||||
Assert.True(Run(options).Succeeded);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("")]
|
||||
[InlineData("otopcua-central-1:9000")] // no scheme
|
||||
[InlineData("/health/ready")] // relative
|
||||
[InlineData("ftp://otopcua-central-1")] // wrong scheme
|
||||
public void NonAbsoluteHttpBaseUrl_IsRejected(string baseUrl)
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.Applications[0].Instances[0].BaseUrl = baseUrl), "BaseUrl");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RelativeManagementUrl_IsRejected()
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.Applications[0].Instances[0].ManagementUrl = "/admin"), "ManagementUrl");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OmittedManagementUrl_IsAllowed()
|
||||
{
|
||||
Assert.True(Run(Valid(o => o.Applications[0].Instances[0].ManagementUrl = null)).Succeeded);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-1)]
|
||||
public void NonPositiveGlobalIntervals_AreRejected(int value)
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.PollIntervalSeconds = value), "PollIntervalSeconds");
|
||||
AssertFailsWith(Valid(o => o.TimeoutSeconds = value), "TimeoutSeconds");
|
||||
AssertFailsWith(Valid(o => o.StaleAfterSeconds = value), "StaleAfterSeconds");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleWindowNotGreaterThanPollInterval_IsRejected()
|
||||
{
|
||||
AssertFailsWith(Valid(o => o.StaleAfterSeconds = o.PollIntervalSeconds), "StaleAfterSeconds");
|
||||
AssertFailsWith(Valid(o => o.StaleAfterSeconds = o.PollIntervalSeconds - 1), "StaleAfterSeconds");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StaleWindowIsCheckedOnEffectiveValues_NotGlobals()
|
||||
{
|
||||
// The globals here are perfectly sane (10 / 45). Only the INSTANCE override inverts the
|
||||
// relationship, which a globals-only check would wave straight through.
|
||||
AssertFailsWith(
|
||||
Valid(o => o.Applications[0].Instances[0].PollIntervalSeconds = 120),
|
||||
"effective StaleAfterSeconds");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ApplicationLevelOverrideCanSatisfyTheStaleRule()
|
||||
{
|
||||
// The mirror of the previous test: an override that RESTORES the relationship must pass, so
|
||||
// the rule is not just "any override fails".
|
||||
var options = Valid(o =>
|
||||
{
|
||||
o.Applications[0].Instances[0].PollIntervalSeconds = 120;
|
||||
o.Applications[0].Instances[0].StaleAfterSeconds = 300;
|
||||
});
|
||||
|
||||
Assert.True(Run(options).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AllFailuresAreAccumulated_NotJustTheFirst()
|
||||
{
|
||||
// OptionsValidatorBase accumulates; an operator fixing a registry one boot at a time is the
|
||||
// failure mode this avoids.
|
||||
var result = Run(Valid(o =>
|
||||
{
|
||||
o.Applications[0].Name = string.Empty;
|
||||
o.Applications[0].Instances[0].BaseUrl = "not-a-url";
|
||||
o.PollIntervalSeconds = 0;
|
||||
}));
|
||||
|
||||
Assert.True(result.Failed);
|
||||
Assert.True(result.Failures.Count() >= 3, $"expected >= 3 failures, got {result.Failures.Count()}");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, null, null, 10, 3, 45)] // all defaults
|
||||
[InlineData(20, null, null, 20, 3, 45)] // application override
|
||||
[InlineData(20, 30, null, 30, 3, 45)] // instance beats application
|
||||
[InlineData(null, 30, 90, 30, 90, 45)] // independent fields resolve independently
|
||||
public void EffectiveTimings_ResolveInstanceOverApplicationOverGlobal(
|
||||
int? applicationPoll, int? instancePoll, int? instanceTimeout,
|
||||
int expectedPoll, int expectedTimeout, int expectedStale)
|
||||
{
|
||||
var options = Valid(o =>
|
||||
{
|
||||
o.Applications[0].PollIntervalSeconds = applicationPoll;
|
||||
o.Applications[0].Instances[0].PollIntervalSeconds = instancePoll;
|
||||
o.Applications[0].Instances[0].TimeoutSeconds = instanceTimeout;
|
||||
});
|
||||
|
||||
var timings = EffectiveTimings.Resolve(options, options.Applications[0], options.Applications[0].Instances[0]);
|
||||
|
||||
Assert.Equal(TimeSpan.FromSeconds(expectedPoll), timings.PollInterval);
|
||||
Assert.Equal(TimeSpan.FromSeconds(expectedTimeout), timings.Timeout);
|
||||
Assert.Equal(TimeSpan.FromSeconds(expectedStale), timings.StaleAfter);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Status derivation and two-strike flap damping. These rules decide what an operator sees, so
|
||||
/// every branch is pinned: the difference between Down and Unreachable sends someone to a
|
||||
/// different place entirely, and damping that fires the wrong way either hides a real outage or
|
||||
/// cries wolf on every dropped packet.
|
||||
/// </summary>
|
||||
public class StatusDerivationTests
|
||||
{
|
||||
private static HealthProbeResult Reachable(string reportStatus, int statusCode) =>
|
||||
new(true, statusCode, new ZbHealthReport { Status = reportStatus }, TimeSpan.FromMilliseconds(5), null);
|
||||
|
||||
private static HealthProbeResult Unreachable(string error) =>
|
||||
new(false, null, null, TimeSpan.FromMilliseconds(5), error);
|
||||
|
||||
[Theory]
|
||||
[InlineData("Healthy", 200, InstanceStatus.Up)]
|
||||
[InlineData("Degraded", 200, InstanceStatus.Degraded)]
|
||||
[InlineData("Unhealthy", 503, InstanceStatus.Down)]
|
||||
public void Observe_MapsCanonicalStatuses(string reportStatus, int code, InstanceStatus expected)
|
||||
{
|
||||
Assert.Equal(expected, StatusDerivation.Observe(Reachable(reportStatus, code)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Observe_TransportFailure_IsUnreachable_NotDown()
|
||||
{
|
||||
// The distinction is the whole point: Down means "the app answered and said it is sick",
|
||||
// Unreachable means "nothing answered" — usually VPN, firewall or a registry typo.
|
||||
Assert.Equal(InstanceStatus.Unreachable, StatusDerivation.Observe(Unreachable("Connection refused")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData("Weird")]
|
||||
public void Observe_UnrecognisedStatus_IsUnreachable(string? reportStatus)
|
||||
{
|
||||
// Parsed as JSON but not this contract — something else is on that port.
|
||||
Assert.Equal(InstanceStatus.Unreachable, StatusDerivation.Observe(Reachable(reportStatus!, 200)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, 200, ActiveState.NotApplicable)]
|
||||
[InlineData(false, 503, ActiveState.NotApplicable)]
|
||||
public void ObserveActive_WithoutActiveRole_IsNotApplicable(bool hasRole, int code, ActiveState expected)
|
||||
{
|
||||
Assert.Equal(expected, StatusDerivation.ObserveActive(hasRole, Reachable("Healthy", code)));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(200, ActiveState.Active)]
|
||||
[InlineData(503, ActiveState.Standby)]
|
||||
[InlineData(404, ActiveState.Unknown)]
|
||||
public void ObserveActive_MapsStatusCodes(int code, ActiveState expected)
|
||||
{
|
||||
Assert.Equal(expected, StatusDerivation.ObserveActive(true, Reachable("Healthy", code)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ObserveActive_NoAnswer_IsUnknown_NotStandby()
|
||||
{
|
||||
// Guessing Standby on a failed probe would let a pair render with zero Active nodes during
|
||||
// a blip, or — worse, if guessed the other way — with two.
|
||||
Assert.Equal(ActiveState.Unknown, StatusDerivation.ObserveActive(true, Unreachable("timed out")));
|
||||
Assert.Equal(ActiveState.Unknown, StatusDerivation.ObserveActive(true, probe: null));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_FirstFailureFromGoodState_HoldsThePreviousStatus()
|
||||
{
|
||||
var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 0, InstanceStatus.Unreachable);
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, status);
|
||||
Assert.Equal(1, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_SecondConsecutiveFailure_Flips()
|
||||
{
|
||||
var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 1, InstanceStatus.Unreachable);
|
||||
|
||||
Assert.Equal(InstanceStatus.Unreachable, status);
|
||||
Assert.Equal(2, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_RecoveryIsImmediate_EvenAfterManyFailures()
|
||||
{
|
||||
var (status, failures) = StatusDerivation.Damp(InstanceStatus.Down, 27, InstanceStatus.Up);
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, status);
|
||||
Assert.Equal(0, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_FirstEverPollThatFails_IsAdoptedImmediately()
|
||||
{
|
||||
// No previous state means nothing to protect and nothing better to show. Holding "Up" here
|
||||
// would be inventing a healthy reading that was never observed.
|
||||
var (status, failures) = StatusDerivation.Damp(previous: null, 0, InstanceStatus.Unreachable);
|
||||
|
||||
Assert.Equal(InstanceStatus.Unreachable, status);
|
||||
Assert.Equal(1, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_AlternatingFailures_NeverFlip()
|
||||
{
|
||||
// The flapping link this exists for: fail, recover, fail, recover. The counter must reset
|
||||
// on every good poll, so the card stays Up instead of oscillating.
|
||||
InstanceStatus? status = InstanceStatus.Up;
|
||||
var failures = 0;
|
||||
|
||||
foreach (var observed in new[]
|
||||
{
|
||||
InstanceStatus.Unreachable, InstanceStatus.Up,
|
||||
InstanceStatus.Unreachable, InstanceStatus.Up,
|
||||
InstanceStatus.Unreachable, InstanceStatus.Up,
|
||||
})
|
||||
{
|
||||
(var next, failures) = StatusDerivation.Damp(status, failures, observed);
|
||||
status = next;
|
||||
}
|
||||
|
||||
Assert.Equal(InstanceStatus.Up, status);
|
||||
Assert.Equal(0, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_DegradedIsNotAFailure_AndIsAdoptedImmediately()
|
||||
{
|
||||
// Degraded is a real answer from a live app, not a failed probe: it must show at once, and
|
||||
// must not accumulate strikes toward Down.
|
||||
var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 1, InstanceStatus.Degraded);
|
||||
|
||||
Assert.Equal(InstanceStatus.Degraded, status);
|
||||
Assert.Equal(0, failures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Damp_TwoDifferentFailureKindsInARow_StillFlips()
|
||||
{
|
||||
// Down then Unreachable is two consecutive failures, not one of each kind restarting the
|
||||
// count — the instance is not answering healthily either way.
|
||||
var (first, failures) = StatusDerivation.Damp(InstanceStatus.Up, 0, InstanceStatus.Down);
|
||||
Assert.Equal(InstanceStatus.Up, first);
|
||||
|
||||
var (second, _) = StatusDerivation.Damp(first, failures, InstanceStatus.Unreachable);
|
||||
Assert.Equal(InstanceStatus.Unreachable, second);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<!--
|
||||
Test stack mirrors ZB.MOM.WW.Theme's (xunit 2.9.3 + bunit), so component tests here read the
|
||||
same way as the kit's own.
|
||||
-->
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
<PackageReference Include="bunit" Version="1.40.0" />
|
||||
<!-- Transitive-security pin, test-only: bunit 1.40.0 pulls AngleSharp 1.2.0, which carries
|
||||
NU1902 (GHSA-pgww-w46g-26qg) and therefore breaks the warnings-as-errors gate. Pinning the
|
||||
patched version here is the same fix HistorianGateway applied (6bc005d); AngleSharp never
|
||||
reaches the shipped app, only the bUnit renderer. -->
|
||||
<PackageReference Include="AngleSharp" Version="1.5.2" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.7" />
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\ZB.MOM.WW.Overview\ZB.MOM.WW.Overview.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,199 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using ZB.MOM.WW.Overview.Polling;
|
||||
|
||||
namespace ZB.MOM.WW.Overview.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Golden-payload tests for the health client. The fixtures are hand-written to the exact shape
|
||||
/// <c>ZbHealthWriter</c> emits — including an entry WITH <c>data</c> (0.2.0+) and one WITHOUT
|
||||
/// (every check that publishes none, and every pre-0.2.0 node), because tolerating both is what
|
||||
/// lets the dashboard run against a fleet that is only partly bumped.
|
||||
/// </summary>
|
||||
public class ZbHealthReportClientTests
|
||||
{
|
||||
private const string ReadyHealthyWithData = """
|
||||
{
|
||||
"status": "Healthy",
|
||||
"totalDurationMs": 12.34,
|
||||
"entries": {
|
||||
"akka-cluster": {
|
||||
"status": "Healthy",
|
||||
"description": "Akka cluster member status: Up",
|
||||
"durationMs": 1.23,
|
||||
"data": {
|
||||
"selfAddress": "akka.tcp://scadabridge@site-a-a:8082",
|
||||
"selfRoles": ["Site", "site-a"],
|
||||
"memberCount": 2,
|
||||
"unreachableCount": 0,
|
||||
"leader": "akka.tcp://scadabridge@site-a-a:8082"
|
||||
}
|
||||
},
|
||||
"localdb": {
|
||||
"status": "Healthy",
|
||||
"description": "Site LocalDb reachable.",
|
||||
"durationMs": 0.45
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
private const string ReadyUnhealthyNoData = """
|
||||
{
|
||||
"status": "Unhealthy",
|
||||
"totalDurationMs": 3001.5,
|
||||
"entries": {
|
||||
"database": {
|
||||
"status": "Unhealthy",
|
||||
"description": null,
|
||||
"durationMs": 3000.1
|
||||
}
|
||||
}
|
||||
}
|
||||
""";
|
||||
|
||||
private static ZbHealthReportClient ClientReturning(
|
||||
HttpStatusCode statusCode,
|
||||
string body,
|
||||
string contentType = "application/json") =>
|
||||
new(new HttpClient(new StubHandler((statusCode, body, contentType))));
|
||||
|
||||
private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(3);
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_HealthyBody_ParsesEnvelopeAndEntries()
|
||||
{
|
||||
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.True(result.Reachable);
|
||||
Assert.Equal(200, result.StatusCode);
|
||||
Assert.Equal("Healthy", result.Report!.Status);
|
||||
Assert.Equal(12.34, result.Report.TotalDurationMs);
|
||||
Assert.Equal(new[] { "akka-cluster", "localdb" }, result.Report.Entries.Keys.OrderBy(k => k, StringComparer.Ordinal));
|
||||
Assert.Equal("Site LocalDb reachable.", result.Report.Entries["localdb"].Description);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_EntryWithData_ExposesLeaderAndCounts()
|
||||
{
|
||||
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
var entry = result.Report!.Entries["akka-cluster"];
|
||||
|
||||
Assert.Equal("akka.tcp://scadabridge@site-a-a:8082", entry.DataString("leader"));
|
||||
Assert.Equal(2, entry.DataInt("memberCount"));
|
||||
Assert.Equal(0, entry.DataInt("unreachableCount"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_EntryWithoutData_ReadsAsNullNotAsAnError()
|
||||
{
|
||||
// The 0.1.0-era / no-data case. It must degrade to "no leader shown", never to a parse
|
||||
// failure — that tolerance is what keeps the dashboard usable mid-rollout.
|
||||
var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData);
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
var entry = result.Report!.Entries["localdb"];
|
||||
|
||||
Assert.True(result.Reachable);
|
||||
Assert.Null(entry.Data);
|
||||
Assert.Null(entry.DataString("leader"));
|
||||
Assert.Null(entry.DataInt("memberCount"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_503_IsAParseableAnswer_NotAFailedProbe()
|
||||
{
|
||||
// 503 carries a full body naming the failing check — the single most useful payload the
|
||||
// dashboard receives. Treating it as a transport failure would throw that away.
|
||||
var client = ClientReturning(HttpStatusCode.ServiceUnavailable, ReadyUnhealthyNoData);
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.True(result.Reachable);
|
||||
Assert.Equal(503, result.StatusCode);
|
||||
Assert.Equal("Unhealthy", result.Report!.Status);
|
||||
Assert.Null(result.Report.Entries["database"].Description);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("<html><body>502 Bad Gateway</body></html>")]
|
||||
[InlineData("not json at all")]
|
||||
public async Task Probe_NonJsonBody_IsUnreachable(string body)
|
||||
{
|
||||
var client = ClientReturning(HttpStatusCode.OK, body, contentType: "text/html");
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Null(result.Report);
|
||||
Assert.NotNull(result.Error);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_ConnectionRefused_IsUnreachableAndDoesNotThrow()
|
||||
{
|
||||
var client = new ZbHealthReportClient(
|
||||
new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused"))));
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout);
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Contains("refused", result.Error, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_Timeout_IsUnreachableAndReportsTheTimeout()
|
||||
{
|
||||
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
|
||||
|
||||
var result = await client.ProbeAsync("http://node/health/ready", TimeSpan.FromMilliseconds(50));
|
||||
|
||||
Assert.False(result.Reachable);
|
||||
Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_HostShutdown_PropagatesCancellation()
|
||||
{
|
||||
// A shutdown must NOT be recorded as a timed-out instance — the sweep is being abandoned,
|
||||
// and writing "Unreachable" for every node on the way out would be a lie in the snapshot.
|
||||
using var shutdown = new CancellationTokenSource();
|
||||
await shutdown.CancelAsync();
|
||||
var client = new ZbHealthReportClient(new HttpClient(new HangingHandler()));
|
||||
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(
|
||||
() => client.ProbeAsync("http://node/health/ready", ProbeTimeout, shutdown.Token));
|
||||
}
|
||||
|
||||
private sealed class StubHandler((HttpStatusCode Status, string Body, string ContentType) response)
|
||||
: HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(new HttpResponseMessage(response.Status)
|
||||
{
|
||||
Content = new StringContent(response.Body, Encoding.UTF8, response.ContentType),
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler
|
||||
{
|
||||
protected override Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken) =>
|
||||
Task.FromException<HttpResponseMessage>(exception);
|
||||
}
|
||||
|
||||
private sealed class HangingHandler : HttpMessageHandler
|
||||
{
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false);
|
||||
return new HttpResponseMessage(HttpStatusCode.OK);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user