merge: observability (register rows 33, 54, 56, 63, 66) — LocalDb backlog KPI history + sustained-threshold badge, honest stream-subscriber gauge rename + real data-connection gauges, central metrics port on the rig, ASPNETCORE_ENVIRONMENT on site containers, import-session rejection counter
This commit is contained in:
@@ -170,6 +170,17 @@
|
||||
IsAvailable="@_sfBufferDepthAvailable"
|
||||
ErrorMessage="@_sfBufferDepthError" />
|
||||
</div>
|
||||
@* Register row 33. The point of charting the backlog is retrospective:
|
||||
the daily purge burst is expected, so what an operator needs is to see
|
||||
whether a past spike DRAINED — which a point-in-time tile can never
|
||||
answer. An unreplicated site emits no samples, so this chart is
|
||||
legitimately empty there rather than a flat, misleading zero line. *@
|
||||
<div class="col-lg-3 col-md-6">
|
||||
<KpiTrendChart Title="LocalDb Replication Backlog"
|
||||
Points="@_localDbBacklogSeries"
|
||||
IsAvailable="@_localDbBacklogAvailable"
|
||||
ErrorMessage="@_localDbBacklogError" />
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
@@ -248,6 +259,14 @@
|
||||
Metrics stale
|
||||
</span>
|
||||
}
|
||||
@if (IsLocalDbBacklogSustained(state))
|
||||
{
|
||||
<span class="badge bg-warning text-dark"
|
||||
data-test="localdb-backlog-badge"
|
||||
title="LocalDb replication oplog backlog has stayed above @HealthOptions.Value.LocalDbBacklogThreshold for over @BacklogSustainedDisplay. A short daily spike at the site_events purge is expected and is NOT this — the peer has stopped draining.">
|
||||
Replication backlog
|
||||
</span>
|
||||
}
|
||||
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
|
||||
{
|
||||
<small class="text-muted">offline since @changedAt.ToString("u")</small>
|
||||
@@ -526,6 +545,27 @@
|
||||
private string StaleTimeoutDisplay =>
|
||||
FormatDuration(HealthOptions.Value.MetricsStaleTimeout);
|
||||
|
||||
// Human-friendly rendering of the configured sustained-backlog dwell for the
|
||||
// "Replication backlog" badge tooltip (e.g. "15 minutes").
|
||||
private string BacklogSustainedDisplay =>
|
||||
FormatDuration(HealthOptions.Value.LocalDbBacklogSustainedFor);
|
||||
|
||||
// Register row 33: the sustained LocalDb replication backlog badge.
|
||||
//
|
||||
// Reads the anchor the aggregator maintains (LocalDbBacklogAboveSince) rather than the
|
||||
// instantaneous backlog on the report, because a bare "backlog > threshold" badge would
|
||||
// fire on EVERY replicated site EVERY day: the site_events retention purge is CDC-captured
|
||||
// like any other write, so the backlog jumps by the deleted batch size and then drains.
|
||||
// Healthy is "the backlog comes back down"; only a dwell can express that.
|
||||
//
|
||||
// A site that reports no backlog at all (unreplicated — rig site-b/site-c — or a failed
|
||||
// poll) never gets an anchor, so it can never be badged. That is the whole null-semantics
|
||||
// requirement: null means "no reading", not zero.
|
||||
private bool IsLocalDbBacklogSustained(SiteHealthState state) =>
|
||||
state.IsOnline
|
||||
&& state.LocalDbBacklogAboveSince is { } since
|
||||
&& DateTimeOffset.UtcNow - since >= HealthOptions.Value.LocalDbBacklogSustainedFor;
|
||||
|
||||
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
|
||||
// Fewer than two means no standby, which disables the manual-failover control. This is
|
||||
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
|
||||
@@ -616,6 +656,9 @@
|
||||
private IReadOnlyList<KpiSeriesPoint>? _sfBufferDepthSeries;
|
||||
private bool _sfBufferDepthAvailable = true;
|
||||
private string? _sfBufferDepthError;
|
||||
private IReadOnlyList<KpiSeriesPoint>? _localDbBacklogSeries;
|
||||
private bool _localDbBacklogAvailable = true;
|
||||
private string? _localDbBacklogError;
|
||||
|
||||
private static bool SiteHasActiveErrors(SiteHealthState state)
|
||||
{
|
||||
@@ -739,6 +782,8 @@
|
||||
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.ScriptErrors, siteId, fromUtc, toUtc);
|
||||
(_sfBufferDepthSeries, _sfBufferDepthAvailable, _sfBufferDepthError) =
|
||||
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.SfBufferDepth, siteId, fromUtc, toUtc);
|
||||
(_localDbBacklogSeries, _localDbBacklogAvailable, _localDbBacklogError) =
|
||||
await LoadTrendSeriesAsync(KpiMetrics.SiteHealth.LocalDbOplogBacklog, siteId, fromUtc, toUtc);
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user