diff --git a/ZB.MOM.WW.Overview/Directory.Build.props b/ZB.MOM.WW.Overview/Directory.Build.props new file mode 100644 index 0000000..26208c2 --- /dev/null +++ b/ZB.MOM.WW.Overview/Directory.Build.props @@ -0,0 +1,32 @@ + + + + + net10.0 + enable + enable + latest + + + + + true + latest + true + true + + + + + false + + + diff --git a/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.slnx b/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.slnx new file mode 100644 index 0000000..c35418c --- /dev/null +++ b/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/ZB.MOM.WW.Overview/nuget.config b/ZB.MOM.WW.Overview/nuget.config new file mode 100644 index 0000000..88c7e71 --- /dev/null +++ b/ZB.MOM.WW.Overview/nuget.config @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/InstanceStatus.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/InstanceStatus.cs new file mode 100644 index 0000000..90b011c --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/InstanceStatus.cs @@ -0,0 +1,132 @@ +namespace ZB.MOM.WW.Overview.Polling; + +/// The derived state of one probed instance. +public enum InstanceStatus +{ + /// Ready tier answered 200 / Healthy. + Up, + + /// Ready tier answered 200 / Degraded — serving, but a check is unhappy. + Degraded, + + /// + /// Ready tier answered 503 / Unhealthy. The app is up and talking; a dependency is not. + /// + Down, + + /// + /// Nothing answered — timeout, connection refused, DNS failure, or a body that is not the + /// canonical health JSON. Kept distinct from because it usually means + /// network / VPN / registry typo rather than a sick application, and the two lead an operator + /// to look in completely different places. + /// + Unreachable, +} + +/// Whether a probed instance is the active half of its pair. +public enum ActiveState +{ + /// The instance has no active/standby concept (single-instance gateways). + NotApplicable, + + /// The active probe did not answer, so activity is unknown. + Unknown, + + /// Active tier answered 200 — this is the active node. + Active, + + /// Active tier answered 503 — this is the standby. + Standby, +} + +/// +/// Pure status derivation: probe result → , with two-strike flap +/// damping applied across polls. Static and dependency-free so every rule is table-testable. +/// +public static class StatusDerivation +{ + /// + /// Consecutive failing observations required before a previously-good instance is shown as + /// failing. A single blip — one dropped packet, one GC pause past the timeout — is not news; + /// two in a row is. + /// + public const int FailureStrikes = 2; + + /// Classifies one ready-tier probe, before any damping. + /// The probe result. + /// The observed status. + public static InstanceStatus Observe(HealthProbeResult probe) + { + ArgumentNullException.ThrowIfNull(probe); + + if (!probe.Reachable || probe.Report is null) + return InstanceStatus.Unreachable; + + // The reported status wins over the HTTP code: they agree by construction (Healthy/Degraded + // → 200, Unhealthy → 503), and the body is the more specific of the two — 200 alone cannot + // tell Up from Degraded. + return probe.Report.Status switch + { + "Healthy" => InstanceStatus.Up, + "Degraded" => InstanceStatus.Degraded, + "Unhealthy" => InstanceStatus.Down, + + // Parsed as JSON but carrying no status we recognise: the endpoint is answering with + // something that is not this contract, which is the Unreachable case, not a sick app. + _ => InstanceStatus.Unreachable, + }; + } + + /// Classifies one active-tier probe. + /// Whether this instance participates in an active/standby pair. + /// The active-tier probe result, or null when not probed. + /// The derived active state. + public static ActiveState ObserveActive(bool hasActiveRole, HealthProbeResult? probe) + { + if (!hasActiveRole) + return ActiveState.NotApplicable; + + if (probe is null || !probe.Reachable || probe.StatusCode is null) + return ActiveState.Unknown; + + return probe.StatusCode switch + { + 200 => ActiveState.Active, + 503 => ActiveState.Standby, + _ => ActiveState.Unknown, + }; + } + + /// + /// Applies two-strike damping: a failing observation only replaces a previously-good state + /// after consecutive failures, while any good observation is + /// adopted immediately. + /// + /// The last displayed status, or null on the first-ever poll. + /// Consecutive failures recorded before this poll. + /// This poll's observed status. + /// The status to display and the updated consecutive-failure count. + /// + /// Recovery is deliberately asymmetric — instant. Damping exists to suppress false alarms, and + /// making recovery equally slow would just convert them into false alarms that linger. + /// + public static (InstanceStatus Status, int ConsecutiveFailures) Damp( + InstanceStatus? previous, + int previousConsecutiveFailures, + InstanceStatus observed) + { + var failing = observed is InstanceStatus.Down or InstanceStatus.Unreachable; + + if (!failing) + return (observed, 0); + + var failures = previousConsecutiveFailures + 1; + + // With no previous state there is nothing to protect and nothing better to show: adopt the + // observation. Claiming Up on a first poll that failed would be a fabrication. + if (previous is null || failures >= FailureStrikes) + return (observed, failures); + + return (previous.Value, failures); + } +} diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReport.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReport.cs new file mode 100644 index 0000000..fb23d70 --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReport.cs @@ -0,0 +1,84 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ZB.MOM.WW.Overview.Polling; + +/// +/// The canonical ZbHealthWriter body every family app serves from MapZbHealth: +/// { status, totalDurationMs, entries: { name: { status, description, durationMs, data? } } }. +/// +/// +/// This mirrors a contract owned by ZB.MOM.WW.Health. It lives in the app for v1 by +/// deliberate decision (design §8.2): promoting it to a ZB.MOM.WW.Health.Client package is +/// worth a publish only once a second consumer exists. +/// +public sealed class ZbHealthReport +{ + /// Aggregate tier status: Healthy, Degraded or Unhealthy. + [JsonPropertyName("status")] + public string? Status { get; init; } + + /// Total time the tier's checks took, in milliseconds. + [JsonPropertyName("totalDurationMs")] + public double TotalDurationMs { get; init; } + + /// Per-check results, keyed by check name (verbatim, e.g. akka-cluster). + [JsonPropertyName("entries")] + public Dictionary Entries { get; init; } = new(StringComparer.Ordinal); +} + +/// One check's result inside a . +public sealed class ZbHealthEntry +{ + /// This check's status: Healthy, Degraded or Unhealthy. + [JsonPropertyName("status")] + public string? Status { get; init; } + + /// Human-readable detail. Always present in the canonical shape; may be JSON null. + [JsonPropertyName("description")] + public string? Description { get; init; } + + /// How long this check took, in milliseconds. + [JsonPropertyName("durationMs")] + public double DurationMs { get; init; } + + /// + /// Optional structured data the check published (ZB.MOM.WW.Health 0.2.0+). Absent entirely on + /// checks that publish none and on every pre-0.2.0 node — hence nullable, and hence the + /// dashboard must render fully (minus leader) without it. + /// + /// + /// Values stay as : this is an open contract carrying arbitrary JSON, + /// and binding it to concrete types here would break the moment a check adds a field. + /// + [JsonPropertyName("data")] + public Dictionary? Data { get; init; } + + /// + /// Reads from as a string, or null when the key is + /// absent or is not a JSON string. + /// + /// The data key, matched verbatim. + /// The string value, or null. + public string? DataString(string key) + { + if (Data is null || !Data.TryGetValue(key, out var value)) + return null; + + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + /// + /// Reads from as an integer, or null when the key is + /// absent or is not a JSON number. + /// + /// The data key, matched verbatim. + /// The integer value, or null. + public int? DataInt(string key) + { + if (Data is null || !Data.TryGetValue(key, out var value)) + return null; + + return value.ValueKind == JsonValueKind.Number && value.TryGetInt32(out var parsed) ? parsed : null; + } +} diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReportClient.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReportClient.cs new file mode 100644 index 0000000..61a1363 --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Polling/ZbHealthReportClient.cs @@ -0,0 +1,109 @@ +using System.Diagnostics; +using System.Text.Json; + +namespace ZB.MOM.WW.Overview.Polling; + +/// +/// One probe of one health endpoint: whether anything answered, what it answered, and how long it +/// took. +/// +/// +/// True when the endpoint answered with a parseable canonical body. False for timeouts, refused +/// connections, DNS failures, and bodies that are not the canonical health JSON. +/// +/// The HTTP status code, when one was received. +/// The parsed body, when it parsed. +/// Wall-clock time for the request. +/// A short reason when is false. +public sealed record HealthProbeResult( + bool Reachable, + int? StatusCode, + ZbHealthReport? Report, + TimeSpan Latency, + string? Error); + +/// +/// GETs a family health endpoint and parses the canonical ZbHealthWriter body. +/// +/// +/// Both 200 and 503 are treated as SUCCESSFUL probes: an Unhealthy tier still returns a full body, +/// and that body is exactly what the dashboard needs to show which check failed. Only a genuine +/// non-answer (transport failure) or an unparseable body is a failed probe. +/// +public sealed class ZbHealthReportClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + private readonly HttpClient _httpClient; + + /// Initializes a new . + /// The HTTP client used for probes. + public ZbHealthReportClient(HttpClient httpClient) => + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + + /// Probes and parses the response. + /// Absolute URL of a health endpoint. + /// Per-request timeout. + /// Token cancelled when the host is shutting down. + /// The probe result; this never throws for an unreachable endpoint. + public async Task ProbeAsync( + string url, + TimeSpan timeout, + CancellationToken cancellationToken = default) + { + var startedAt = Stopwatch.GetTimestamp(); + + // Linked, not a bare timeout token: shutdown must cancel an in-flight probe immediately, + // and the two causes have to stay distinguishable so a shutdown is not misreported as a + // timed-out instance. + using var timeoutSource = new CancellationTokenSource(timeout); + using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutSource.Token); + + try + { + using var response = await _httpClient + .GetAsync(url, HttpCompletionOption.ResponseContentRead, linked.Token) + .ConfigureAwait(false); + + var body = await response.Content.ReadAsStringAsync(linked.Token).ConfigureAwait(false); + var statusCode = (int)response.StatusCode; + + ZbHealthReport? report; + try + { + report = JsonSerializer.Deserialize(body, SerializerOptions); + } + catch (JsonException ex) + { + // Answered, but not with this contract — a reverse proxy error page, an HTML login + // redirect, a wrong port. That is a registry/plumbing problem, not a sick app, so + // it lands as Unreachable rather than Down. + return Failed(startedAt, statusCode, $"response body is not canonical health JSON: {ex.Message}"); + } + + if (report is null) + return Failed(startedAt, statusCode, "response body deserialized to null"); + + return new HealthProbeResult(true, statusCode, report, Stopwatch.GetElapsedTime(startedAt), null); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Host shutdown, not an instance fault — let the caller abandon the sweep. + throw; + } + catch (OperationCanceledException) + { + return Failed(startedAt, null, $"timed out after {timeout.TotalSeconds:0.##}s"); + } + catch (HttpRequestException ex) + { + return Failed(startedAt, null, ex.Message); + } + } + + private static HealthProbeResult Failed(long startedAt, int? statusCode, string error) => + new(false, statusCode, null, Stopwatch.GetElapsedTime(startedAt), error); +} diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Program.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Program.cs new file mode 100644 index 0000000..5e7e4f8 --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Program.cs @@ -0,0 +1,28 @@ +using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.Overview.Registry; + +var builder = WebApplication.CreateBuilder(args); + +// Pre-host presence check. The validator below covers shape and consistency, but it only runs once +// the options are resolved — a registry section that is missing entirely binds to an empty +// OverviewOptions, and the failure a reader can act on is "you configured no registry", not +// "Applications must have at least 1 entry". Fail here, before the host is built, with the key name. +ConfigPreflight.For(builder.Configuration) + .RequireValue($"{OverviewOptions.SectionName}:Applications:0:Name") + .RequireValue($"{OverviewOptions.SectionName}:Applications:0:Instances:0:BaseUrl") + .ThrowIfInvalid(); + +builder.Services.AddValidatedOptions( + builder.Configuration, OverviewOptions.SectionName); + +var app = builder.Build(); + +app.MapGet("/", () => "ZB.MOM.WW.Overview"); + +await app.RunAsync(); + +/// +/// Exposed so WebApplicationFactory<Program> can boot the real host in tests +/// (top-level statements otherwise compile to an internal entry-point class). +/// +public partial class Program; diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptions.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptions.cs new file mode 100644 index 0000000..127aa27 --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptions.cs @@ -0,0 +1,124 @@ +namespace ZB.MOM.WW.Overview.Registry; + +/// +/// The dashboard's registry: which application instances to probe, and how often. Bound from the +/// Overview configuration section and validated by . +/// +/// +/// Every instance is probed the same way — GET {BaseUrl}/health/ready, plus +/// {BaseUrl}/health/active when . There is +/// deliberately no per-instance probe-mode switch: one probe model, one status derivation. +/// +public sealed class OverviewOptions +{ + /// The configuration section this binds to. + public const string SectionName = "Overview"; + + /// How often the poller sweeps every instance, in seconds. + public int PollIntervalSeconds { get; set; } = 10; + + /// Per-request timeout, in seconds. + public int TimeoutSeconds { get; set; } = 3; + + /// + /// How old a snapshot may get before its instance renders as Stale (out of date / offline) + /// regardless of its last known status, in seconds. This is what catches a stuck poller, a + /// paused refresh or a slept machine — not just failed probes — so it must exceed the poll + /// interval or every instance would read Stale between two healthy polls. + /// + public int StaleAfterSeconds { get; set; } = 45; + + /// The registered applications, each with one or more instances. + public IList Applications { get; set; } = new List(); +} + +/// One registered application (a section on the dashboard). +public sealed class ApplicationEntry +{ + /// Display name, unique across the registry. + public string Name { get; set; } = string.Empty; + + /// Label for the per-instance management link, e.g. "AdminUI" or "Dashboard". + public string ManagementLabel { get; set; } = "UI"; + + /// Overrides for this application. + public int? PollIntervalSeconds { get; set; } + + /// Overrides for this application. + public int? TimeoutSeconds { get; set; } + + /// Overrides for this application. + public int? StaleAfterSeconds { get; set; } + + /// This application's instances; at least one is required. + public IList Instances { get; set; } = new List(); +} + +/// One probed instance (a card on the dashboard). +public sealed class InstanceEntry +{ + /// Display name, unique within its application. + public string Name { get; set; } = string.Empty; + + /// + /// The cluster group this instance belongs to (e.g. "central", "site-a"). One Akka cluster is + /// one group: leader aggregation and the split-brain check are scoped to it. Null for + /// standalone instances. + /// + public string? Group { get; set; } + + /// Absolute base URL whose /health/* endpoints are probed. + public string BaseUrl { get; set; } = string.Empty; + + /// Absolute URL of this instance's own management UI, linked from its card. + public string? ManagementUrl { get; set; } + + /// + /// Whether this instance participates in an active/standby pair. False for the single-instance + /// gateways, which have no such concept — their cards show status without an Active badge and + /// are never probed for /health/active. + /// + public bool HasActiveRole { get; set; } + + /// Overrides the application and global poll interval for this instance. + public int? PollIntervalSeconds { get; set; } + + /// Overrides the application and global request timeout for this instance. + public int? TimeoutSeconds { get; set; } + + /// Overrides the application and global staleness window for this instance. + public int? StaleAfterSeconds { get; set; } +} + +/// +/// The effective (override-resolved) timings for one instance. Instance wins over application, +/// application wins over global — resolved once here so no caller re-implements the precedence. +/// +/// Effective poll cadence. +/// Effective per-request timeout. +/// Effective staleness window. +public readonly record struct EffectiveTimings(TimeSpan PollInterval, TimeSpan Timeout, TimeSpan StaleAfter) +{ + /// Resolves the effective timings for . + /// The global registry defaults. + /// The instance's application. + /// The instance itself. + /// The resolved timings. + public static EffectiveTimings Resolve( + OverviewOptions options, + ApplicationEntry application, + InstanceEntry instance) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(application); + ArgumentNullException.ThrowIfNull(instance); + + return new EffectiveTimings( + TimeSpan.FromSeconds( + instance.PollIntervalSeconds ?? application.PollIntervalSeconds ?? options.PollIntervalSeconds), + TimeSpan.FromSeconds( + instance.TimeoutSeconds ?? application.TimeoutSeconds ?? options.TimeoutSeconds), + TimeSpan.FromSeconds( + instance.StaleAfterSeconds ?? application.StaleAfterSeconds ?? options.StaleAfterSeconds)); + } +} diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptionsValidator.cs b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptionsValidator.cs new file mode 100644 index 0000000..b6e1274 --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/Registry/OverviewOptionsValidator.cs @@ -0,0 +1,109 @@ +using ZB.MOM.WW.Configuration; + +namespace ZB.MOM.WW.Overview.Registry; + +/// +/// Validates the Overview registry. Registered through +/// AddValidatedOptions<OverviewOptions, OverviewOptionsValidator>, so a malformed +/// registry fails the host at startup rather than surfacing as a silently-empty dashboard or a +/// first-poll exception. +/// +public sealed class OverviewOptionsValidator : OptionsValidatorBase +{ + /// + protected override void Validate(ValidationBuilder builder, OverviewOptions options) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(options); + + RequirePositive(builder, options.PollIntervalSeconds, "Overview:PollIntervalSeconds"); + RequirePositive(builder, options.TimeoutSeconds, "Overview:TimeoutSeconds"); + RequirePositive(builder, options.StaleAfterSeconds, "Overview:StaleAfterSeconds"); + + builder.MinCount(options.Applications as IReadOnlyCollection, 1, "Overview:Applications"); + + var seenApplications = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var a = 0; a < options.Applications.Count; a++) + { + var application = options.Applications[a]; + var appPath = $"Overview:Applications:{a}"; + + builder.Required(application.Name, $"{appPath}:Name"); + if (!string.IsNullOrWhiteSpace(application.Name) && !seenApplications.Add(application.Name)) + builder.Add($"{appPath}:Name '{application.Name}' is duplicated; application names must be unique"); + + RequirePositiveIfSet(builder, application.PollIntervalSeconds, $"{appPath}:PollIntervalSeconds"); + RequirePositiveIfSet(builder, application.TimeoutSeconds, $"{appPath}:TimeoutSeconds"); + RequirePositiveIfSet(builder, application.StaleAfterSeconds, $"{appPath}:StaleAfterSeconds"); + + builder.MinCount( + application.Instances as IReadOnlyCollection, 1, $"{appPath}:Instances"); + + var seenInstances = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var i = 0; i < application.Instances.Count; i++) + { + var instance = application.Instances[i]; + var path = $"{appPath}:Instances:{i}"; + + builder.Required(instance.Name, $"{path}:Name"); + if (!string.IsNullOrWhiteSpace(instance.Name) && !seenInstances.Add(instance.Name)) + { + builder.Add( + $"{path}:Name '{instance.Name}' is duplicated within application " + + $"'{application.Name}'; instance names must be unique per application"); + } + + RequireAbsoluteHttpUrl(builder, instance.BaseUrl, $"{path}:BaseUrl", required: true); + RequireAbsoluteHttpUrl(builder, instance.ManagementUrl, $"{path}:ManagementUrl", required: false); + + RequirePositiveIfSet(builder, instance.PollIntervalSeconds, $"{path}:PollIntervalSeconds"); + RequirePositiveIfSet(builder, instance.TimeoutSeconds, $"{path}:TimeoutSeconds"); + RequirePositiveIfSet(builder, instance.StaleAfterSeconds, $"{path}:StaleAfterSeconds"); + + // Checked on the EFFECTIVE values, not the globals: an override at either level can + // invert the relationship on one instance while the global pair stays sane, and an + // instance whose stale window is inside its poll interval would render Stale + // between two perfectly healthy polls. + var timings = EffectiveTimings.Resolve(options, application, instance); + if (timings.StaleAfter <= timings.PollInterval) + { + builder.Add( + $"{path}: effective StaleAfterSeconds ({timings.StaleAfter.TotalSeconds:0.##}) must be " + + $"greater than effective PollIntervalSeconds ({timings.PollInterval.TotalSeconds:0.##}) " + + "— otherwise this instance renders Stale between two healthy polls"); + } + } + } + } + + private static void RequirePositive(ValidationBuilder builder, int value, string field) => + builder.RequireThat(value > 0, $"{field} must be greater than 0"); + + private static void RequirePositiveIfSet(ValidationBuilder builder, int? value, string field) + { + if (value is { } set) + RequirePositive(builder, set, field); + } + + private static void RequireAbsoluteHttpUrl( + ValidationBuilder builder, string? value, string field, bool required) + { + if (string.IsNullOrWhiteSpace(value)) + { + if (required) + builder.Add($"{field} is required"); + return; + } + + // Absolute + http(s) explicitly: a relative URL binds without complaint and only fails at + // the first poll, as an opaque HttpClient error against a dashboard that has no base + // address of its own. + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) + || (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)) + { + builder.Add($"{field} must be an absolute http:// or https:// URL (was '{value}')"); + } + } +} diff --git a/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj new file mode 100644 index 0000000..d8df41c --- /dev/null +++ b/ZB.MOM.WW.Overview/src/ZB.MOM.WW.Overview/ZB.MOM.WW.Overview.csproj @@ -0,0 +1,38 @@ + + + + + net10.0 + ZB.MOM.WW.Overview + zb-mom-ww-overview + + + + + + + + + + + + + + + + + + + diff --git a/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/OverviewOptionsValidatorTests.cs b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/OverviewOptionsValidatorTests.cs new file mode 100644 index 0000000..ae5aab8 --- /dev/null +++ b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/OverviewOptionsValidatorTests.cs @@ -0,0 +1,226 @@ +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Overview.Registry; + +namespace ZB.MOM.WW.Overview.Tests; + +/// +/// Registry validation. A malformed registry must fail the host at startup with a message naming +/// the offending key — the alternative is a dashboard that boots and quietly shows nothing, or one +/// that throws on its first poll where nobody is looking. +/// +public class OverviewOptionsValidatorTests +{ + private static readonly OverviewOptionsValidator Validator = new(); + + private static OverviewOptions Valid(Action? mutate = null) + { + var options = new OverviewOptions + { + PollIntervalSeconds = 10, + TimeoutSeconds = 3, + StaleAfterSeconds = 45, + Applications = + { + new ApplicationEntry + { + Name = "OtOpcUa", + ManagementLabel = "AdminUI", + Instances = + { + new InstanceEntry + { + Name = "central-1", + Group = "central", + BaseUrl = "http://otopcua-central-1:9000", + ManagementUrl = "http://otopcua-central-1:9000/", + HasActiveRole = true, + }, + }, + }, + }, + }; + + mutate?.Invoke(options); + return options; + } + + private static ValidateOptionsResult Run(OverviewOptions options) => Validator.Validate(null, options); + + private static void AssertFailsWith(OverviewOptions options, string expectedFragment) + { + var result = Run(options); + + Assert.True(result.Failed, "expected validation to fail"); + Assert.Contains( + result.Failures, + f => f.Contains(expectedFragment, StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void ValidRegistry_Succeeds() + { + // Positive control: without this, every "fails with X" assertion below could be passing + // because the fixture itself is malformed for some unrelated reason. + Assert.True(Run(Valid()).Succeeded); + } + + [Fact] + public void EmptyRegistry_IsRejected() + { + AssertFailsWith(Valid(o => o.Applications.Clear()), "Overview:Applications"); + } + + [Fact] + public void ApplicationWithNoInstances_IsRejected() + { + AssertFailsWith(Valid(o => o.Applications[0].Instances.Clear()), "Instances"); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public void ApplicationWithoutName_IsRejected(string name) + { + AssertFailsWith(Valid(o => o.Applications[0].Name = name), "Applications:0:Name"); + } + + [Fact] + public void DuplicateApplicationNames_AreRejected() + { + AssertFailsWith( + Valid(o => o.Applications.Add(new ApplicationEntry + { + Name = "OtOpcUa", + Instances = { new InstanceEntry { Name = "other", BaseUrl = "http://other:9000" } }, + })), + "duplicated"); + } + + [Fact] + public void DuplicateInstanceNamesWithinAnApplication_AreRejected() + { + AssertFailsWith( + Valid(o => o.Applications[0].Instances.Add(new InstanceEntry + { + Name = "central-1", + BaseUrl = "http://otopcua-central-2:9000", + })), + "duplicated"); + } + + [Fact] + public void SameInstanceNameInDifferentApplications_IsAllowed() + { + // "central-1" under OtOpcUa and "central-1" under ScadaBridge are different machines with + // the same conventional name — uniqueness is per application, not global. + var options = Valid(o => o.Applications.Add(new ApplicationEntry + { + Name = "ScadaBridge", + Instances = { new InstanceEntry { Name = "central-1", BaseUrl = "http://sb-central-a:5000" } }, + })); + + Assert.True(Run(options).Succeeded); + } + + [Theory] + [InlineData("")] + [InlineData("otopcua-central-1:9000")] // no scheme + [InlineData("/health/ready")] // relative + [InlineData("ftp://otopcua-central-1")] // wrong scheme + public void NonAbsoluteHttpBaseUrl_IsRejected(string baseUrl) + { + AssertFailsWith(Valid(o => o.Applications[0].Instances[0].BaseUrl = baseUrl), "BaseUrl"); + } + + [Fact] + public void RelativeManagementUrl_IsRejected() + { + AssertFailsWith(Valid(o => o.Applications[0].Instances[0].ManagementUrl = "/admin"), "ManagementUrl"); + } + + [Fact] + public void OmittedManagementUrl_IsAllowed() + { + Assert.True(Run(Valid(o => o.Applications[0].Instances[0].ManagementUrl = null)).Succeeded); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void NonPositiveGlobalIntervals_AreRejected(int value) + { + AssertFailsWith(Valid(o => o.PollIntervalSeconds = value), "PollIntervalSeconds"); + AssertFailsWith(Valid(o => o.TimeoutSeconds = value), "TimeoutSeconds"); + AssertFailsWith(Valid(o => o.StaleAfterSeconds = value), "StaleAfterSeconds"); + } + + [Fact] + public void StaleWindowNotGreaterThanPollInterval_IsRejected() + { + AssertFailsWith(Valid(o => o.StaleAfterSeconds = o.PollIntervalSeconds), "StaleAfterSeconds"); + AssertFailsWith(Valid(o => o.StaleAfterSeconds = o.PollIntervalSeconds - 1), "StaleAfterSeconds"); + } + + [Fact] + public void StaleWindowIsCheckedOnEffectiveValues_NotGlobals() + { + // The globals here are perfectly sane (10 / 45). Only the INSTANCE override inverts the + // relationship, which a globals-only check would wave straight through. + AssertFailsWith( + Valid(o => o.Applications[0].Instances[0].PollIntervalSeconds = 120), + "effective StaleAfterSeconds"); + } + + [Fact] + public void ApplicationLevelOverrideCanSatisfyTheStaleRule() + { + // The mirror of the previous test: an override that RESTORES the relationship must pass, so + // the rule is not just "any override fails". + var options = Valid(o => + { + o.Applications[0].Instances[0].PollIntervalSeconds = 120; + o.Applications[0].Instances[0].StaleAfterSeconds = 300; + }); + + Assert.True(Run(options).Succeeded); + } + + [Fact] + public void AllFailuresAreAccumulated_NotJustTheFirst() + { + // OptionsValidatorBase accumulates; an operator fixing a registry one boot at a time is the + // failure mode this avoids. + var result = Run(Valid(o => + { + o.Applications[0].Name = string.Empty; + o.Applications[0].Instances[0].BaseUrl = "not-a-url"; + o.PollIntervalSeconds = 0; + })); + + Assert.True(result.Failed); + Assert.True(result.Failures.Count() >= 3, $"expected >= 3 failures, got {result.Failures.Count()}"); + } + + [Theory] + [InlineData(null, null, null, 10, 3, 45)] // all defaults + [InlineData(20, null, null, 20, 3, 45)] // application override + [InlineData(20, 30, null, 30, 3, 45)] // instance beats application + [InlineData(null, 30, 90, 30, 90, 45)] // independent fields resolve independently + public void EffectiveTimings_ResolveInstanceOverApplicationOverGlobal( + int? applicationPoll, int? instancePoll, int? instanceTimeout, + int expectedPoll, int expectedTimeout, int expectedStale) + { + var options = Valid(o => + { + o.Applications[0].PollIntervalSeconds = applicationPoll; + o.Applications[0].Instances[0].PollIntervalSeconds = instancePoll; + o.Applications[0].Instances[0].TimeoutSeconds = instanceTimeout; + }); + + var timings = EffectiveTimings.Resolve(options, options.Applications[0], options.Applications[0].Instances[0]); + + Assert.Equal(TimeSpan.FromSeconds(expectedPoll), timings.PollInterval); + Assert.Equal(TimeSpan.FromSeconds(expectedTimeout), timings.Timeout); + Assert.Equal(TimeSpan.FromSeconds(expectedStale), timings.StaleAfter); + } +} diff --git a/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/StatusDerivationTests.cs b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/StatusDerivationTests.cs new file mode 100644 index 0000000..e551214 --- /dev/null +++ b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/StatusDerivationTests.cs @@ -0,0 +1,155 @@ +using ZB.MOM.WW.Overview.Polling; + +namespace ZB.MOM.WW.Overview.Tests; + +/// +/// Status derivation and two-strike flap damping. These rules decide what an operator sees, so +/// every branch is pinned: the difference between Down and Unreachable sends someone to a +/// different place entirely, and damping that fires the wrong way either hides a real outage or +/// cries wolf on every dropped packet. +/// +public class StatusDerivationTests +{ + private static HealthProbeResult Reachable(string reportStatus, int statusCode) => + new(true, statusCode, new ZbHealthReport { Status = reportStatus }, TimeSpan.FromMilliseconds(5), null); + + private static HealthProbeResult Unreachable(string error) => + new(false, null, null, TimeSpan.FromMilliseconds(5), error); + + [Theory] + [InlineData("Healthy", 200, InstanceStatus.Up)] + [InlineData("Degraded", 200, InstanceStatus.Degraded)] + [InlineData("Unhealthy", 503, InstanceStatus.Down)] + public void Observe_MapsCanonicalStatuses(string reportStatus, int code, InstanceStatus expected) + { + Assert.Equal(expected, StatusDerivation.Observe(Reachable(reportStatus, code))); + } + + [Fact] + public void Observe_TransportFailure_IsUnreachable_NotDown() + { + // The distinction is the whole point: Down means "the app answered and said it is sick", + // Unreachable means "nothing answered" — usually VPN, firewall or a registry typo. + Assert.Equal(InstanceStatus.Unreachable, StatusDerivation.Observe(Unreachable("Connection refused"))); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("Weird")] + public void Observe_UnrecognisedStatus_IsUnreachable(string? reportStatus) + { + // Parsed as JSON but not this contract — something else is on that port. + Assert.Equal(InstanceStatus.Unreachable, StatusDerivation.Observe(Reachable(reportStatus!, 200))); + } + + [Theory] + [InlineData(false, 200, ActiveState.NotApplicable)] + [InlineData(false, 503, ActiveState.NotApplicable)] + public void ObserveActive_WithoutActiveRole_IsNotApplicable(bool hasRole, int code, ActiveState expected) + { + Assert.Equal(expected, StatusDerivation.ObserveActive(hasRole, Reachable("Healthy", code))); + } + + [Theory] + [InlineData(200, ActiveState.Active)] + [InlineData(503, ActiveState.Standby)] + [InlineData(404, ActiveState.Unknown)] + public void ObserveActive_MapsStatusCodes(int code, ActiveState expected) + { + Assert.Equal(expected, StatusDerivation.ObserveActive(true, Reachable("Healthy", code))); + } + + [Fact] + public void ObserveActive_NoAnswer_IsUnknown_NotStandby() + { + // Guessing Standby on a failed probe would let a pair render with zero Active nodes during + // a blip, or — worse, if guessed the other way — with two. + Assert.Equal(ActiveState.Unknown, StatusDerivation.ObserveActive(true, Unreachable("timed out"))); + Assert.Equal(ActiveState.Unknown, StatusDerivation.ObserveActive(true, probe: null)); + } + + [Fact] + public void Damp_FirstFailureFromGoodState_HoldsThePreviousStatus() + { + var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 0, InstanceStatus.Unreachable); + + Assert.Equal(InstanceStatus.Up, status); + Assert.Equal(1, failures); + } + + [Fact] + public void Damp_SecondConsecutiveFailure_Flips() + { + var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 1, InstanceStatus.Unreachable); + + Assert.Equal(InstanceStatus.Unreachable, status); + Assert.Equal(2, failures); + } + + [Fact] + public void Damp_RecoveryIsImmediate_EvenAfterManyFailures() + { + var (status, failures) = StatusDerivation.Damp(InstanceStatus.Down, 27, InstanceStatus.Up); + + Assert.Equal(InstanceStatus.Up, status); + Assert.Equal(0, failures); + } + + [Fact] + public void Damp_FirstEverPollThatFails_IsAdoptedImmediately() + { + // No previous state means nothing to protect and nothing better to show. Holding "Up" here + // would be inventing a healthy reading that was never observed. + var (status, failures) = StatusDerivation.Damp(previous: null, 0, InstanceStatus.Unreachable); + + Assert.Equal(InstanceStatus.Unreachable, status); + Assert.Equal(1, failures); + } + + [Fact] + public void Damp_AlternatingFailures_NeverFlip() + { + // The flapping link this exists for: fail, recover, fail, recover. The counter must reset + // on every good poll, so the card stays Up instead of oscillating. + InstanceStatus? status = InstanceStatus.Up; + var failures = 0; + + foreach (var observed in new[] + { + InstanceStatus.Unreachable, InstanceStatus.Up, + InstanceStatus.Unreachable, InstanceStatus.Up, + InstanceStatus.Unreachable, InstanceStatus.Up, + }) + { + (var next, failures) = StatusDerivation.Damp(status, failures, observed); + status = next; + } + + Assert.Equal(InstanceStatus.Up, status); + Assert.Equal(0, failures); + } + + [Fact] + public void Damp_DegradedIsNotAFailure_AndIsAdoptedImmediately() + { + // Degraded is a real answer from a live app, not a failed probe: it must show at once, and + // must not accumulate strikes toward Down. + var (status, failures) = StatusDerivation.Damp(InstanceStatus.Up, 1, InstanceStatus.Degraded); + + Assert.Equal(InstanceStatus.Degraded, status); + Assert.Equal(0, failures); + } + + [Fact] + public void Damp_TwoDifferentFailureKindsInARow_StillFlips() + { + // Down then Unreachable is two consecutive failures, not one of each kind restarting the + // count — the instance is not answering healthily either way. + var (first, failures) = StatusDerivation.Damp(InstanceStatus.Up, 0, InstanceStatus.Down); + Assert.Equal(InstanceStatus.Up, first); + + var (second, _) = StatusDerivation.Damp(first, failures, InstanceStatus.Unreachable); + Assert.Equal(InstanceStatus.Unreachable, second); + } +} diff --git a/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZB.MOM.WW.Overview.Tests.csproj b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZB.MOM.WW.Overview.Tests.csproj new file mode 100644 index 0000000..dbd6fb4 --- /dev/null +++ b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZB.MOM.WW.Overview.Tests.csproj @@ -0,0 +1,33 @@ + + + + + false + + + + + + + + + + + + + + + + + + + + + + diff --git a/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZbHealthReportClientTests.cs b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZbHealthReportClientTests.cs new file mode 100644 index 0000000..d68f6b4 --- /dev/null +++ b/ZB.MOM.WW.Overview/tests/ZB.MOM.WW.Overview.Tests/ZbHealthReportClientTests.cs @@ -0,0 +1,199 @@ +using System.Net; +using System.Text; +using ZB.MOM.WW.Overview.Polling; + +namespace ZB.MOM.WW.Overview.Tests; + +/// +/// Golden-payload tests for the health client. The fixtures are hand-written to the exact shape +/// ZbHealthWriter emits — including an entry WITH data (0.2.0+) and one WITHOUT +/// (every check that publishes none, and every pre-0.2.0 node), because tolerating both is what +/// lets the dashboard run against a fleet that is only partly bumped. +/// +public class ZbHealthReportClientTests +{ + private const string ReadyHealthyWithData = """ + { + "status": "Healthy", + "totalDurationMs": 12.34, + "entries": { + "akka-cluster": { + "status": "Healthy", + "description": "Akka cluster member status: Up", + "durationMs": 1.23, + "data": { + "selfAddress": "akka.tcp://scadabridge@site-a-a:8082", + "selfRoles": ["Site", "site-a"], + "memberCount": 2, + "unreachableCount": 0, + "leader": "akka.tcp://scadabridge@site-a-a:8082" + } + }, + "localdb": { + "status": "Healthy", + "description": "Site LocalDb reachable.", + "durationMs": 0.45 + } + } + } + """; + + private const string ReadyUnhealthyNoData = """ + { + "status": "Unhealthy", + "totalDurationMs": 3001.5, + "entries": { + "database": { + "status": "Unhealthy", + "description": null, + "durationMs": 3000.1 + } + } + } + """; + + private static ZbHealthReportClient ClientReturning( + HttpStatusCode statusCode, + string body, + string contentType = "application/json") => + new(new HttpClient(new StubHandler((statusCode, body, contentType)))); + + private static readonly TimeSpan ProbeTimeout = TimeSpan.FromSeconds(3); + + [Fact] + public async Task Probe_HealthyBody_ParsesEnvelopeAndEntries() + { + var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + + Assert.True(result.Reachable); + Assert.Equal(200, result.StatusCode); + Assert.Equal("Healthy", result.Report!.Status); + Assert.Equal(12.34, result.Report.TotalDurationMs); + Assert.Equal(new[] { "akka-cluster", "localdb" }, result.Report.Entries.Keys.OrderBy(k => k, StringComparer.Ordinal)); + Assert.Equal("Site LocalDb reachable.", result.Report.Entries["localdb"].Description); + } + + [Fact] + public async Task Probe_EntryWithData_ExposesLeaderAndCounts() + { + var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + var entry = result.Report!.Entries["akka-cluster"]; + + Assert.Equal("akka.tcp://scadabridge@site-a-a:8082", entry.DataString("leader")); + Assert.Equal(2, entry.DataInt("memberCount")); + Assert.Equal(0, entry.DataInt("unreachableCount")); + } + + [Fact] + public async Task Probe_EntryWithoutData_ReadsAsNullNotAsAnError() + { + // The 0.1.0-era / no-data case. It must degrade to "no leader shown", never to a parse + // failure — that tolerance is what keeps the dashboard usable mid-rollout. + var client = ClientReturning(HttpStatusCode.OK, ReadyHealthyWithData); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + var entry = result.Report!.Entries["localdb"]; + + Assert.True(result.Reachable); + Assert.Null(entry.Data); + Assert.Null(entry.DataString("leader")); + Assert.Null(entry.DataInt("memberCount")); + } + + [Fact] + public async Task Probe_503_IsAParseableAnswer_NotAFailedProbe() + { + // 503 carries a full body naming the failing check — the single most useful payload the + // dashboard receives. Treating it as a transport failure would throw that away. + var client = ClientReturning(HttpStatusCode.ServiceUnavailable, ReadyUnhealthyNoData); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + + Assert.True(result.Reachable); + Assert.Equal(503, result.StatusCode); + Assert.Equal("Unhealthy", result.Report!.Status); + Assert.Null(result.Report.Entries["database"].Description); + } + + [Theory] + [InlineData("502 Bad Gateway")] + [InlineData("not json at all")] + public async Task Probe_NonJsonBody_IsUnreachable(string body) + { + var client = ClientReturning(HttpStatusCode.OK, body, contentType: "text/html"); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + + Assert.False(result.Reachable); + Assert.Null(result.Report); + Assert.NotNull(result.Error); + } + + [Fact] + public async Task Probe_ConnectionRefused_IsUnreachableAndDoesNotThrow() + { + var client = new ZbHealthReportClient( + new HttpClient(new ThrowingHandler(new HttpRequestException("Connection refused")))); + + var result = await client.ProbeAsync("http://node/health/ready", ProbeTimeout); + + Assert.False(result.Reachable); + Assert.Contains("refused", result.Error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Probe_Timeout_IsUnreachableAndReportsTheTimeout() + { + var client = new ZbHealthReportClient(new HttpClient(new HangingHandler())); + + var result = await client.ProbeAsync("http://node/health/ready", TimeSpan.FromMilliseconds(50)); + + Assert.False(result.Reachable); + Assert.Contains("timed out", result.Error, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task Probe_HostShutdown_PropagatesCancellation() + { + // A shutdown must NOT be recorded as a timed-out instance — the sweep is being abandoned, + // and writing "Unreachable" for every node on the way out would be a lie in the snapshot. + using var shutdown = new CancellationTokenSource(); + await shutdown.CancelAsync(); + var client = new ZbHealthReportClient(new HttpClient(new HangingHandler())); + + await Assert.ThrowsAnyAsync( + () => client.ProbeAsync("http://node/health/ready", ProbeTimeout, shutdown.Token)); + } + + private sealed class StubHandler((HttpStatusCode Status, string Body, string ContentType) response) + : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(response.Status) + { + Content = new StringContent(response.Body, Encoding.UTF8, response.ContentType), + }); + } + + private sealed class ThrowingHandler(Exception exception) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromException(exception); + } + + private sealed class HangingHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(HttpStatusCode.OK); + } + } +}