merge: observability (register rows 33, 54, 56, 63, 66) — LocalDb backlog KPI history + sustained-threshold badge, honest stream-subscriber gauge rename + real data-connection gauges, central metrics port on the rig, ASPNETCORE_ENVIRONMENT on site containers, import-session rejection counter

This commit is contained in:
Joseph Doherty
2026-08-15 17:56:14 -04:00
38 changed files with 1417 additions and 52 deletions
@@ -170,6 +170,17 @@
IsAvailable="@_sfBufferDepthAvailable"
ErrorMessage="@_sfBufferDepthError" />
</div>
@* Register row 33. The point of charting the backlog is retrospective:
the daily purge burst is expected, so what an operator needs is to see
whether a past spike DRAINED — which a point-in-time tile can never
answer. An unreplicated site emits no samples, so this chart is
legitimately empty there rather than a flat, misleading zero line. *@
<div class="col-lg-3 col-md-6">
<KpiTrendChart Title="LocalDb Replication Backlog"
Points="@_localDbBacklogSeries"
IsAvailable="@_localDbBacklogAvailable"
ErrorMessage="@_localDbBacklogError" />
</div>
</div>
}
</div>
@@ -248,6 +259,14 @@
Metrics stale
</span>
}
@if (IsLocalDbBacklogSustained(state))
{
<span class="badge bg-warning text-dark"
data-test="localdb-backlog-badge"
title="LocalDb replication oplog backlog has stayed above @HealthOptions.Value.LocalDbBacklogThreshold for over @BacklogSustainedDisplay. A short daily spike at the site_events purge is expected and is NOT this — the peer has stopped draining.">
Replication backlog
</span>
}
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
{
<small class="text-muted">offline since @changedAt.ToString("u")</small>
@@ -526,6 +545,27 @@
private string StaleTimeoutDisplay =>
FormatDuration(HealthOptions.Value.MetricsStaleTimeout);
// Human-friendly rendering of the configured sustained-backlog dwell for the
// "Replication backlog" badge tooltip (e.g. "15 minutes").
private string BacklogSustainedDisplay =>
FormatDuration(HealthOptions.Value.LocalDbBacklogSustainedFor);
// Register row 33: the sustained LocalDb replication backlog badge.
//
// Reads the anchor the aggregator maintains (LocalDbBacklogAboveSince) rather than the
// instantaneous backlog on the report, because a bare "backlog > threshold" badge would
// fire on EVERY replicated site EVERY day: the site_events retention purge is CDC-captured
// like any other write, so the backlog jumps by the deleted batch size and then drains.
// Healthy is "the backlog comes back down"; only a dwell can express that.
//
// A site that reports no backlog at all (unreplicated — rig site-b/site-c — or a failed
// poll) never gets an anchor, so it can never be badged. That is the whole null-semantics
// requirement: null means "no reading", not zero.
private bool IsLocalDbBacklogSustained(SiteHealthState state) =>
state.IsOnline
&& state.LocalDbBacklogAboveSince is { } since
&& DateTimeOffset.UtcNow - since >= HealthOptions.Value.LocalDbBacklogSustainedFor;
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
// Fewer than two means no standby, which disables the manual-failover control. This is
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
@@ -616,6 +656,9 @@
private IReadOnlyList<KpiSeriesPoint>? _sfBufferDepthSeries;
private bool _sfBufferDepthAvailable = true;
private string? _sfBufferDepthError;
private IReadOnlyList<KpiSeriesPoint>? _localDbBacklogSeries;
private bool _localDbBacklogAvailable = true;
private string? _localDbBacklogError;
private static bool SiteHasActiveErrors(SiteHealthState state)
{
@@ -739,6 +782,8 @@
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc);
(_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) =
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc);
(_localDbBacklogSeries, _localDbBacklogAvailable, _localDbBacklogError) =
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.LocalDbOplogBacklog, siteId, fromUtc, toUtc);
}
finally
{