fix(adminui): deployments drift chip never serves a stale "In sync" (#903)
v2-ci / unit-tests (push) Has been cancelled
v2-ci / unit-tests (push) Has been cancelled
Root cause: DeploymentDriftProbe (a singleton shared by every circuit) cached its verdict until the ConfigEdit audit watermark (MAX(EditedAtUtc), COUNT(*)) moved, assuming every configuration write appends a ConfigEdit row. Only deployments and driver restart/reconnect do; the AdminUI editors (RawTreeService, UnsTreeService, ScriptEdit, cluster/node/ACL pages) write the config tables directly. So after a draft change the page kept showing the cached "In sync" until something unrelated moved the watermark. - DeploymentDriftProbe: the watermark cache is gone; every verdict comes from a composition that started after the caller asked. Compositions stay serialised behind the gate, and callers queued behind one share a verdict only when that composition started after they arrived (start sequence number) for the same sealed hash, so sharing is exact, never stale. - Deployments page: the drift check runs in OnAfterRenderAsync (interactive render only, so a visit costs one composition, not two with prerender); the first render shows a neutral "Checking for changes..." chip, never a verdict; a failed check shows "Drift check failed" and logs a warning instead of crashing the page. A deploy re-runs the check. - Tests: probe tests for an un-audited draft change (real ConfigComposer over InMemory), fresh composition per call, and queued callers not reusing an older composition; bUnit page tests that a second page load after a direct draft change shows "Configuration drift" and that the first render shows "Checking" until the comparison answers. All fail against the previous code. DeploymentDriftProbeSqlServerTests is removed: it pinned the SQL translation of the watermark query, which no longer exists. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
@page "/deployments"
|
||||
@using Microsoft.AspNetCore.Components.Authorization
|
||||
@using Microsoft.EntityFrameworkCore
|
||||
@using Microsoft.Extensions.Logging
|
||||
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
|
||||
@using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin
|
||||
@using ZB.MOM.WW.OtOpcUa.Configuration
|
||||
@@ -13,6 +14,7 @@
|
||||
@inject IAdminOperationsClient AdminOps
|
||||
@inject AuthenticationStateProvider AuthState
|
||||
@inject ZB.MOM.WW.OtOpcUa.AdminUI.Deployments.DeploymentDriftProbe DriftProbe
|
||||
@inject ILogger<Deployments> Logger
|
||||
@rendermode InteractiveServer
|
||||
|
||||
<PageErrorBoundary>
|
||||
@@ -25,11 +27,22 @@
|
||||
@(_busy ? "Deploying…" : "Deploy current configuration")
|
||||
</button>
|
||||
|
||||
@if (_drift is not null)
|
||||
@* lmxopcua#903: the chip says nothing about sync until a composition taken after this page opened has
|
||||
answered — first render shows "Checking…", never a verdict carried over from an earlier load. *@
|
||||
@switch (_driftState)
|
||||
{
|
||||
<span class="badge @(_drift.Value ? "bg-warning text-dark" : "bg-success")">
|
||||
@(_drift.Value ? "Configuration drift" : "In sync")
|
||||
</span>
|
||||
case DriftState.Checking:
|
||||
<span class="badge bg-secondary" data-testid="drift-chip" role="status">Checking for changes…</span>
|
||||
break;
|
||||
case DriftState.Drift:
|
||||
<span class="badge bg-warning text-dark" data-testid="drift-chip">Configuration drift</span>
|
||||
break;
|
||||
case DriftState.InSync:
|
||||
<span class="badge bg-success" data-testid="drift-chip">In sync</span>
|
||||
break;
|
||||
case DriftState.Failed:
|
||||
<span class="badge bg-secondary" data-testid="drift-chip" title="See the host log">Drift check failed</span>
|
||||
break;
|
||||
}
|
||||
</div>
|
||||
|
||||
@@ -85,13 +98,54 @@
|
||||
private bool _busy;
|
||||
private bool _lastSuccess;
|
||||
private string? _lastMessage;
|
||||
private bool? _drift;
|
||||
private enum DriftState { None, Checking, Drift, InSync, Failed }
|
||||
|
||||
private DriftState _driftState;
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
await ReloadAsync();
|
||||
}
|
||||
|
||||
protected override async Task OnAfterRenderAsync(bool firstRender)
|
||||
{
|
||||
// Interactive render only (OnAfterRender never runs during prerender), so a page visit pays for one
|
||||
// composition, after the table is already on screen.
|
||||
if (firstRender)
|
||||
{
|
||||
await RefreshDriftAsync();
|
||||
StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RefreshDriftAsync()
|
||||
{
|
||||
// Drift: if no sealed deployment yet, no drift to report. Otherwise the probe compares the latest sealed
|
||||
// revision hash to a fresh composition of the live-edit state.
|
||||
var latestSealed = _deployments.FirstOrDefault(d => d.Status == DeploymentStatus.Sealed);
|
||||
if (latestSealed is null)
|
||||
{
|
||||
_driftState = DriftState.None;
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var drift = await DriftProbe.GetDriftAsync(latestSealed.RevisionHash, CancellationToken.None);
|
||||
_driftState = drift switch
|
||||
{
|
||||
true => DriftState.Drift,
|
||||
false => DriftState.InSync,
|
||||
null => DriftState.None,
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogWarning(ex, "Deployments: drift check against revision {RevisionHash} failed.", latestSealed.RevisionHash);
|
||||
_driftState = DriftState.Failed;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReloadAsync()
|
||||
{
|
||||
await using var db = await DbFactory.CreateDbContextAsync();
|
||||
@@ -109,13 +163,8 @@
|
||||
.Take(50)
|
||||
.ToListAsync();
|
||||
|
||||
// Drift: if no sealed deployment yet, no drift to report. Otherwise the probe compares the
|
||||
// latest sealed revision hash to the live-edit state — recomposing only when the ConfigEdit
|
||||
// audit watermark has moved, so a re-render of this page is not a full fleet snapshot.
|
||||
var latestSealed = _deployments.FirstOrDefault(d => d.Status == DeploymentStatus.Sealed);
|
||||
_drift = latestSealed is null
|
||||
? null
|
||||
: await DriftProbe.GetDriftAsync(latestSealed.RevisionHash, CancellationToken.None);
|
||||
// The verdict for these rows is not known yet; RefreshDriftAsync answers it.
|
||||
_driftState = _deployments.Any(d => d.Status == DeploymentStatus.Sealed) ? DriftState.Checking : DriftState.None;
|
||||
}
|
||||
|
||||
private async Task StartDeploymentAsync()
|
||||
@@ -139,6 +188,7 @@
|
||||
_ => result.Message ?? "Deployment rejected.",
|
||||
};
|
||||
await ReloadAsync();
|
||||
await RefreshDriftAsync();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -5,50 +5,35 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Deployments;
|
||||
|
||||
/// <summary>
|
||||
/// Answers the /deployments page's "Configuration drift" chip without recomposing the whole
|
||||
/// configuration on every page load.
|
||||
/// Answers the /deployments page's "Configuration drift" chip: does the live-edit state still hash to the
|
||||
/// latest sealed revision?
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why.</b> The chip's question — "does the live-edit state still hash to the latest sealed
|
||||
/// revision?" — was answered by calling <c>ConfigComposer.SnapshotAndFlattenAsync</c> on every
|
||||
/// <c>ReloadAsync</c>: eighteen unfiltered table reads, a deterministic JSON serialisation of the
|
||||
/// entire fleet configuration, a gzip pass and a SHA-256, all to render one badge. Opening
|
||||
/// /deployments, deploying, or simply re-rendering the page paid for it again.
|
||||
/// <b>Always a fresh composition (lmxopcua#903).</b> This probe used to cache its verdict until the
|
||||
/// <c>ConfigEdit</c> audit watermark <c>(MAX(EditedAtUtc), COUNT(*))</c> moved, on the premise that every
|
||||
/// configuration write appends a <c>ConfigEdit</c> row. That premise was false: the AdminUI's own editors
|
||||
/// (<c>RawTreeService</c>, <c>UnsTreeService</c>, <c>ScriptEdit</c>, the cluster/node/ACL pages) write the
|
||||
/// configuration tables directly and append no audit row — only deployments and driver restart/reconnect do.
|
||||
/// So after a draft change the chip kept serving the cached "In sync" until something unrelated moved the
|
||||
/// watermark: an operator could believe there was nothing to deploy. There is no cheap, provider-neutral
|
||||
/// signal that moves on every write to the ~18 composed tables, so every verdict is now computed from a
|
||||
/// composition that read the configuration AFTER the request arrived.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The watermark.</b> <c>ConfigEdit</c> is the append-only audit row that every mutating
|
||||
/// live-edit operation writes (see the entity's remarks — <c>AdminOperationsActor</c> appends one
|
||||
/// per mutation), so <c>(MAX(EditedAtUtc), COUNT(*))</c> moves whenever the configuration could
|
||||
/// have changed. That pair is one indexed aggregate (<c>IX_ConfigEdit_EditedAt</c>) against a
|
||||
/// full recomposition. When neither the watermark nor the sealed revision hash has moved since
|
||||
/// the last verdict, the cached verdict is returned unchanged.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Accepted staleness.</b> An edit path that mutated a configuration table WITHOUT writing a
|
||||
/// <c>ConfigEdit</c> row would not invalidate this cache, and the chip could then read "In sync"
|
||||
/// over a genuinely drifted configuration until the next audited edit. That is acceptable
|
||||
/// precisely because the chip is advisory: the authoritative comparison is
|
||||
/// <c>AdminOperationsActor</c>'s own recomposition at deploy time, which always reads live and
|
||||
/// which reports <c>NoChanges</c> for a deploy that would be a no-op. Today every
|
||||
/// <c>AdminOperationsActor</c> mutation writes the audit row, so the gap is theoretical — but a
|
||||
/// future write path that skips the audit must not be relied on to move this chip. The
|
||||
/// <c>COUNT(*)</c> half is what keeps a same-timestamp edit (two mutations inside one tick, or
|
||||
/// an edit deleted and replaced) from hiding behind an unchanged maximum.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Concurrency.</b> The cheap watermark read runs outside the gate, so concurrent circuits do
|
||||
/// not serialise on it. Only the recompose path takes the <see cref="SemaphoreSlim"/>, and it
|
||||
/// double-checks the cache after acquiring it — so N circuits that miss at the same instant
|
||||
/// share ONE recomposition rather than launching N. A throw leaves the previous verdict in
|
||||
/// place and propagates to its caller alone.
|
||||
/// <b>Concurrency.</b> Compositions are serialised behind a <see cref="SemaphoreSlim"/>, so N circuits
|
||||
/// opening the page at once never run N fleet snapshots in parallel. A caller queued behind a composition
|
||||
/// reuses that verdict only when the composition <i>started after the caller arrived</i> (tracked by a
|
||||
/// start sequence number) and was against the same sealed revision — such a composition has already seen
|
||||
/// every edit committed before the caller asked, so sharing it is exact, not stale. A throw leaves the
|
||||
/// previous verdict in place and propagates to its caller alone.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="dbFactory">Factory for the configuration database context.</param>
|
||||
/// <param name="composeRevisionHash">
|
||||
/// The compose seam — returns the revision hash of a fresh snapshot of the supplied context.
|
||||
/// Defaults to the real <c>ConfigComposer.SnapshotAndFlattenAsync(...).RevisionHash</c>; tests
|
||||
/// inject a counting fake to prove the cache actually elides recompositions.
|
||||
/// inject a counting fake.
|
||||
/// </param>
|
||||
public sealed class DeploymentDriftProbe(
|
||||
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
|
||||
@@ -57,11 +42,14 @@ public sealed class DeploymentDriftProbe(
|
||||
private readonly Func<OtOpcUaConfigDbContext, Task<string>> _composeRevisionHash =
|
||||
composeRevisionHash ?? ComposeRevisionHashAsync;
|
||||
|
||||
private readonly SemaphoreSlim _recomposeGate = new(1, 1);
|
||||
private readonly SemaphoreSlim _composeGate = new(1, 1);
|
||||
|
||||
/// <summary>The last computed verdict, or <see langword="null"/> before the first one. Volatile: the
|
||||
/// fast path reads it off the gate, so the publishing write must be visible to every circuit.</summary>
|
||||
private volatile Verdict? _verdict;
|
||||
/// <summary>How many compositions have started; incremented under the gate as each one begins.</summary>
|
||||
private long _composesStarted;
|
||||
|
||||
/// <summary>The last computed verdict, or <see langword="null"/> before the first one. Read and written
|
||||
/// only under the gate.</summary>
|
||||
private Verdict? _verdict;
|
||||
|
||||
/// <summary>
|
||||
/// Reports whether the live-edit configuration has drifted from the supplied sealed revision.
|
||||
@@ -80,66 +68,37 @@ public sealed class DeploymentDriftProbe(
|
||||
{
|
||||
if (string.IsNullOrEmpty(latestSealedRevisionHash)) return null;
|
||||
|
||||
await using var db = await dbFactory.CreateDbContextAsync(ct);
|
||||
var watermark = await ReadWatermarkAsync(db, ct);
|
||||
// Any composition numbered above this one starts after this point, so it reads every edit
|
||||
// committed before this call.
|
||||
var arrivedAfter = Interlocked.Read(ref _composesStarted);
|
||||
|
||||
// Fast path: no gate, because a hit reads one immutable record reference.
|
||||
if (IsCurrent(_verdict, watermark, latestSealedRevisionHash, out var cached)) return cached;
|
||||
|
||||
await _recomposeGate.WaitAsync(ct);
|
||||
await _composeGate.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
// Double-check: a caller ahead of us in the queue may have just recomposed the same state.
|
||||
if (IsCurrent(_verdict, watermark, latestSealedRevisionHash, out cached)) return cached;
|
||||
var shared = _verdict;
|
||||
if (shared is not null
|
||||
&& shared.Sequence > arrivedAfter
|
||||
&& string.Equals(shared.SealedRevisionHash, latestSealedRevisionHash, StringComparison.Ordinal))
|
||||
{
|
||||
return shared.Drift;
|
||||
}
|
||||
|
||||
var sequence = Interlocked.Increment(ref _composesStarted);
|
||||
await using var db = await dbFactory.CreateDbContextAsync(ct);
|
||||
var current = await _composeRevisionHash(db);
|
||||
var drift = !string.Equals(current, latestSealedRevisionHash, StringComparison.Ordinal);
|
||||
_verdict = new Verdict(watermark, latestSealedRevisionHash, drift);
|
||||
_verdict = new Verdict(sequence, latestSealedRevisionHash, drift);
|
||||
return drift;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_recomposeGate.Release();
|
||||
_composeGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsCurrent(
|
||||
Verdict? verdict, Watermark watermark, string sealedHash, out bool drift)
|
||||
{
|
||||
if (verdict is not null
|
||||
&& verdict.Watermark == watermark
|
||||
&& string.Equals(verdict.SealedRevisionHash, sealedHash, StringComparison.Ordinal))
|
||||
{
|
||||
drift = verdict.Drift;
|
||||
return true;
|
||||
}
|
||||
|
||||
drift = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Reads the audit watermark as a single aggregate row (no rows at all ⇒ the empty watermark).</summary>
|
||||
private static async Task<Watermark> ReadWatermarkAsync(OtOpcUaConfigDbContext db, CancellationToken ct)
|
||||
{
|
||||
// GroupBy(_ => 1) collapses the whole table to one aggregate row, so this is ONE round-trip for
|
||||
// both scalars rather than two. An empty table produces no group at all, so FirstOrDefaultAsync
|
||||
// lands on null and we return the default (never-edited) watermark; the Max is cast to nullable
|
||||
// so the projection stays valid for the no-rows shape either way.
|
||||
var row = await db.ConfigEdits
|
||||
.AsNoTracking()
|
||||
.GroupBy(_ => 1)
|
||||
.Select(g => new { LastEditUtc = g.Max(e => (DateTime?)e.EditedAtUtc), EditCount = g.Count() })
|
||||
.FirstOrDefaultAsync(ct);
|
||||
|
||||
return row is null ? default : new Watermark(row.LastEditUtc, row.EditCount);
|
||||
}
|
||||
|
||||
private static async Task<string> ComposeRevisionHashAsync(OtOpcUaConfigDbContext db) =>
|
||||
(await ConfigComposer.SnapshotAndFlattenAsync(db)).RevisionHash;
|
||||
|
||||
/// <summary>The ConfigEdit audit watermark: newest edit timestamp plus the total edit count.</summary>
|
||||
private readonly record struct Watermark(DateTime? LastEditUtc, int EditCount);
|
||||
|
||||
/// <summary>A drift answer, valid only for the watermark and sealed revision it was computed against.</summary>
|
||||
private sealed record Verdict(Watermark Watermark, string SealedRevisionHash, bool Drift);
|
||||
/// <summary>A drift answer from the composition with start number <paramref name="Sequence"/>.</summary>
|
||||
private sealed record Verdict(long Sequence, string SealedRevisionHash, bool Drift);
|
||||
}
|
||||
|
||||
@@ -121,8 +121,8 @@ public static class EndpointRouteBuilderExtensions
|
||||
// TryAdd so a host that already bound a clock (test harness, future fused-host registration) wins.
|
||||
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
|
||||
|
||||
// /deployments drift chip. A singleton so its ConfigEdit-watermark cache is shared across
|
||||
// circuits — otherwise every page load would recompose the whole fleet configuration.
|
||||
// /deployments drift chip. A singleton so concurrent circuits share one composition gate (and a
|
||||
// composition that started after they asked) instead of running N fleet snapshots at once.
|
||||
// Constructed explicitly rather than by ctor scan: the compose seam is an optional parameter
|
||||
// that only tests supply, and production must always get the real ConfigComposer default.
|
||||
services.AddSingleton(sp => new Deployments.DeploymentDriftProbe(
|
||||
|
||||
Reference in New Issue
Block a user