fix(adminui): deployments drift chip never serves a stale "In sync" (#903)
v2-ci / unit-tests (push) Has been cancelled

Root cause: DeploymentDriftProbe (a singleton shared by every circuit) cached its verdict until
the ConfigEdit audit watermark (MAX(EditedAtUtc), COUNT(*)) moved, assuming every configuration
write appends a ConfigEdit row. Only deployments and driver restart/reconnect do; the AdminUI
editors (RawTreeService, UnsTreeService, ScriptEdit, cluster/node/ACL pages) write the config
tables directly. So after a draft change the page kept showing the cached "In sync" until
something unrelated moved the watermark.

- DeploymentDriftProbe: the watermark cache is gone; every verdict comes from a composition that
  started after the caller asked. Compositions stay serialised behind the gate, and callers queued
  behind one share a verdict only when that composition started after they arrived (start
  sequence number) for the same sealed hash, so sharing is exact, never stale.
- Deployments page: the drift check runs in OnAfterRenderAsync (interactive render only, so a
  visit costs one composition, not two with prerender); the first render shows a neutral
  "Checking for changes..." chip, never a verdict; a failed check shows "Drift check failed" and
  logs a warning instead of crashing the page. A deploy re-runs the check.
- Tests: probe tests for an un-audited draft change (real ConfigComposer over InMemory), fresh
  composition per call, and queued callers not reusing an older composition; bUnit page tests that
  a second page load after a direct draft change shows "Configuration drift" and that the first
  render shows "Checking" until the comparison answers. All fail against the previous code.
  DeploymentDriftProbeSqlServerTests is removed: it pinned the SQL translation of the watermark
  query, which no longer exists.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 01:50:37 -05:00
parent b85e004bdb
commit 35576485b1
7 changed files with 317 additions and 143 deletions
@@ -1,6 +1,7 @@
@page "/deployments"
@using Microsoft.AspNetCore.Components.Authorization
@using Microsoft.EntityFrameworkCore
@using Microsoft.Extensions.Logging
@using ZB.MOM.WW.OtOpcUa.Commons.Interfaces
@using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin
@using ZB.MOM.WW.OtOpcUa.Configuration
@@ -13,6 +14,7 @@
@inject IAdminOperationsClient AdminOps
@inject AuthenticationStateProvider AuthState
@inject ZB.MOM.WW.OtOpcUa.AdminUI.Deployments.DeploymentDriftProbe DriftProbe
@inject ILogger<Deployments> Logger
@rendermode InteractiveServer
<PageErrorBoundary>
@@ -25,11 +27,22 @@
@(_busy ? "Deploying…" : "Deploy current configuration")
</button>
@if (_drift is not null)
@* lmxopcua#903: the chip says nothing about sync until a composition taken after this page opened has
answered — first render shows "Checking…", never a verdict carried over from an earlier load. *@
@switch (_driftState)
{
<span class="badge @(_drift.Value ? "bg-warning text-dark" : "bg-success")">
@(_drift.Value ? "Configuration drift" : "In sync")
</span>
case DriftState.Checking:
<span class="badge bg-secondary" data-testid="drift-chip" role="status">Checking for changes…</span>
break;
case DriftState.Drift:
<span class="badge bg-warning text-dark" data-testid="drift-chip">Configuration drift</span>
break;
case DriftState.InSync:
<span class="badge bg-success" data-testid="drift-chip">In sync</span>
break;
case DriftState.Failed:
<span class="badge bg-secondary" data-testid="drift-chip" title="See the host log">Drift check failed</span>
break;
}
</div>
@@ -85,13 +98,54 @@
private bool _busy;
private bool _lastSuccess;
private string? _lastMessage;
private bool? _drift;
private enum DriftState { None, Checking, Drift, InSync, Failed }
private DriftState _driftState;
protected override async Task OnInitializedAsync()
{
await ReloadAsync();
}
protected override async Task OnAfterRenderAsync(bool firstRender)
{
// Interactive render only (OnAfterRender never runs during prerender), so a page visit pays for one
// composition, after the table is already on screen.
if (firstRender)
{
await RefreshDriftAsync();
StateHasChanged();
}
}
private async Task RefreshDriftAsync()
{
// Drift: if no sealed deployment yet, no drift to report. Otherwise the probe compares the latest sealed
// revision hash to a fresh composition of the live-edit state.
var latestSealed = _deployments.FirstOrDefault(d => d.Status == DeploymentStatus.Sealed);
if (latestSealed is null)
{
_driftState = DriftState.None;
return;
}
try
{
var drift = await DriftProbe.GetDriftAsync(latestSealed.RevisionHash, CancellationToken.None);
_driftState = drift switch
{
true => DriftState.Drift,
false => DriftState.InSync,
null => DriftState.None,
};
}
catch (Exception ex)
{
Logger.LogWarning(ex, "Deployments: drift check against revision {RevisionHash} failed.", latestSealed.RevisionHash);
_driftState = DriftState.Failed;
}
}
private async Task ReloadAsync()
{
await using var db = await DbFactory.CreateDbContextAsync();
@@ -109,13 +163,8 @@
.Take(50)
.ToListAsync();
// Drift: if no sealed deployment yet, no drift to report. Otherwise the probe compares the
// latest sealed revision hash to the live-edit state — recomposing only when the ConfigEdit
// audit watermark has moved, so a re-render of this page is not a full fleet snapshot.
var latestSealed = _deployments.FirstOrDefault(d => d.Status == DeploymentStatus.Sealed);
_drift = latestSealed is null
? null
: await DriftProbe.GetDriftAsync(latestSealed.RevisionHash, CancellationToken.None);
// The verdict for these rows is not known yet; RefreshDriftAsync answers it.
_driftState = _deployments.Any(d => d.Status == DeploymentStatus.Sealed) ? DriftState.Checking : DriftState.None;
}
private async Task StartDeploymentAsync()
@@ -139,6 +188,7 @@
_ => result.Message ?? "Deployment rejected.",
};
await ReloadAsync();
await RefreshDriftAsync();
}
catch (Exception ex)
{
@@ -5,50 +5,35 @@ using ZB.MOM.WW.OtOpcUa.ControlPlane.AdminOperations;
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Deployments;
/// <summary>
/// Answers the /deployments page's "Configuration drift" chip without recomposing the whole
/// configuration on every page load.
/// Answers the /deployments page's "Configuration drift" chip: does the live-edit state still hash to the
/// latest sealed revision?
/// </summary>
/// <remarks>
/// <para>
/// <b>Why.</b> The chip's question — "does the live-edit state still hash to the latest sealed
/// revision?" — was answered by calling <c>ConfigComposer.SnapshotAndFlattenAsync</c> on every
/// <c>ReloadAsync</c>: eighteen unfiltered table reads, a deterministic JSON serialisation of the
/// entire fleet configuration, a gzip pass and a SHA-256, all to render one badge. Opening
/// /deployments, deploying, or simply re-rendering the page paid for it again.
/// <b>Always a fresh composition (lmxopcua#903).</b> This probe used to cache its verdict until the
/// <c>ConfigEdit</c> audit watermark <c>(MAX(EditedAtUtc), COUNT(*))</c> moved, on the premise that every
/// configuration write appends a <c>ConfigEdit</c> row. That premise was false: the AdminUI's own editors
/// (<c>RawTreeService</c>, <c>UnsTreeService</c>, <c>ScriptEdit</c>, the cluster/node/ACL pages) write the
/// configuration tables directly and append no audit row — only deployments and driver restart/reconnect do.
/// So after a draft change the chip kept serving the cached "In sync" until something unrelated moved the
/// watermark: an operator could believe there was nothing to deploy. There is no cheap, provider-neutral
/// signal that moves on every write to the ~18 composed tables, so every verdict is now computed from a
/// composition that read the configuration AFTER the request arrived.
/// </para>
/// <para>
/// <b>The watermark.</b> <c>ConfigEdit</c> is the append-only audit row that every mutating
/// live-edit operation writes (see the entity's remarks — <c>AdminOperationsActor</c> appends one
/// per mutation), so <c>(MAX(EditedAtUtc), COUNT(*))</c> moves whenever the configuration could
/// have changed. That pair is one indexed aggregate (<c>IX_ConfigEdit_EditedAt</c>) against a
/// full recomposition. When neither the watermark nor the sealed revision hash has moved since
/// the last verdict, the cached verdict is returned unchanged.
/// </para>
/// <para>
/// <b>Accepted staleness.</b> An edit path that mutated a configuration table WITHOUT writing a
/// <c>ConfigEdit</c> row would not invalidate this cache, and the chip could then read "In sync"
/// over a genuinely drifted configuration until the next audited edit. That is acceptable
/// precisely because the chip is advisory: the authoritative comparison is
/// <c>AdminOperationsActor</c>'s own recomposition at deploy time, which always reads live and
/// which reports <c>NoChanges</c> for a deploy that would be a no-op. Today every
/// <c>AdminOperationsActor</c> mutation writes the audit row, so the gap is theoretical — but a
/// future write path that skips the audit must not be relied on to move this chip. The
/// <c>COUNT(*)</c> half is what keeps a same-timestamp edit (two mutations inside one tick, or
/// an edit deleted and replaced) from hiding behind an unchanged maximum.
/// </para>
/// <para>
/// <b>Concurrency.</b> The cheap watermark read runs outside the gate, so concurrent circuits do
/// not serialise on it. Only the recompose path takes the <see cref="SemaphoreSlim"/>, and it
/// double-checks the cache after acquiring it — so N circuits that miss at the same instant
/// share ONE recomposition rather than launching N. A throw leaves the previous verdict in
/// place and propagates to its caller alone.
/// <b>Concurrency.</b> Compositions are serialised behind a <see cref="SemaphoreSlim"/>, so N circuits
/// opening the page at once never run N fleet snapshots in parallel. A caller queued behind a composition
/// reuses that verdict only when the composition <i>started after the caller arrived</i> (tracked by a
/// start sequence number) and was against the same sealed revision — such a composition has already seen
/// every edit committed before the caller asked, so sharing it is exact, not stale. A throw leaves the
/// previous verdict in place and propagates to its caller alone.
/// </para>
/// </remarks>
/// <param name="dbFactory">Factory for the configuration database context.</param>
/// <param name="composeRevisionHash">
/// The compose seam — returns the revision hash of a fresh snapshot of the supplied context.
/// Defaults to the real <c>ConfigComposer.SnapshotAndFlattenAsync(...).RevisionHash</c>; tests
/// inject a counting fake to prove the cache actually elides recompositions.
/// inject a counting fake.
/// </param>
public sealed class DeploymentDriftProbe(
IDbContextFactory<OtOpcUaConfigDbContext> dbFactory,
@@ -57,11 +42,14 @@ public sealed class DeploymentDriftProbe(
private readonly Func<OtOpcUaConfigDbContext, Task<string>> _composeRevisionHash =
composeRevisionHash ?? ComposeRevisionHashAsync;
private readonly SemaphoreSlim _recomposeGate = new(1, 1);
private readonly SemaphoreSlim _composeGate = new(1, 1);
/// <summary>The last computed verdict, or <see langword="null"/> before the first one. Volatile: the
/// fast path reads it off the gate, so the publishing write must be visible to every circuit.</summary>
private volatile Verdict? _verdict;
/// <summary>How many compositions have started; incremented under the gate as each one begins.</summary>
private long _composesStarted;
/// <summary>The last computed verdict, or <see langword="null"/> before the first one. Read and written
/// only under the gate.</summary>
private Verdict? _verdict;
/// <summary>
/// Reports whether the live-edit configuration has drifted from the supplied sealed revision.
@@ -80,66 +68,37 @@ public sealed class DeploymentDriftProbe(
{
if (string.IsNullOrEmpty(latestSealedRevisionHash)) return null;
await using var db = await dbFactory.CreateDbContextAsync(ct);
var watermark = await ReadWatermarkAsync(db, ct);
// Any composition numbered above this one starts after this point, so it reads every edit
// committed before this call.
var arrivedAfter = Interlocked.Read(ref _composesStarted);
// Fast path: no gate, because a hit reads one immutable record reference.
if (IsCurrent(_verdict, watermark, latestSealedRevisionHash, out var cached)) return cached;
await _recomposeGate.WaitAsync(ct);
await _composeGate.WaitAsync(ct);
try
{
// Double-check: a caller ahead of us in the queue may have just recomposed the same state.
if (IsCurrent(_verdict, watermark, latestSealedRevisionHash, out cached)) return cached;
var shared = _verdict;
if (shared is not null
&& shared.Sequence > arrivedAfter
&& string.Equals(shared.SealedRevisionHash, latestSealedRevisionHash, StringComparison.Ordinal))
{
return shared.Drift;
}
var sequence = Interlocked.Increment(ref _composesStarted);
await using var db = await dbFactory.CreateDbContextAsync(ct);
var current = await _composeRevisionHash(db);
var drift = !string.Equals(current, latestSealedRevisionHash, StringComparison.Ordinal);
_verdict = new Verdict(watermark, latestSealedRevisionHash, drift);
_verdict = new Verdict(sequence, latestSealedRevisionHash, drift);
return drift;
}
finally
{
_recomposeGate.Release();
_composeGate.Release();
}
}
private static bool IsCurrent(
Verdict? verdict, Watermark watermark, string sealedHash, out bool drift)
{
if (verdict is not null
&& verdict.Watermark == watermark
&& string.Equals(verdict.SealedRevisionHash, sealedHash, StringComparison.Ordinal))
{
drift = verdict.Drift;
return true;
}
drift = false;
return false;
}
/// <summary>Reads the audit watermark as a single aggregate row (no rows at all ⇒ the empty watermark).</summary>
private static async Task<Watermark> ReadWatermarkAsync(OtOpcUaConfigDbContext db, CancellationToken ct)
{
// GroupBy(_ => 1) collapses the whole table to one aggregate row, so this is ONE round-trip for
// both scalars rather than two. An empty table produces no group at all, so FirstOrDefaultAsync
// lands on null and we return the default (never-edited) watermark; the Max is cast to nullable
// so the projection stays valid for the no-rows shape either way.
var row = await db.ConfigEdits
.AsNoTracking()
.GroupBy(_ => 1)
.Select(g => new { LastEditUtc = g.Max(e => (DateTime?)e.EditedAtUtc), EditCount = g.Count() })
.FirstOrDefaultAsync(ct);
return row is null ? default : new Watermark(row.LastEditUtc, row.EditCount);
}
private static async Task<string> ComposeRevisionHashAsync(OtOpcUaConfigDbContext db) =>
(await ConfigComposer.SnapshotAndFlattenAsync(db)).RevisionHash;
/// <summary>The ConfigEdit audit watermark: newest edit timestamp plus the total edit count.</summary>
private readonly record struct Watermark(DateTime? LastEditUtc, int EditCount);
/// <summary>A drift answer, valid only for the watermark and sealed revision it was computed against.</summary>
private sealed record Verdict(Watermark Watermark, string SealedRevisionHash, bool Drift);
/// <summary>A drift answer from the composition with start number <paramref name="Sequence"/>.</summary>
private sealed record Verdict(long Sequence, string SealedRevisionHash, bool Drift);
}
@@ -121,8 +121,8 @@ public static class EndpointRouteBuilderExtensions
// TryAdd so a host that already bound a clock (test harness, future fused-host registration) wins.
services.TryAddSingleton<TimeProvider>(TimeProvider.System);
// /deployments drift chip. A singleton so its ConfigEdit-watermark cache is shared across
// circuits — otherwise every page load would recompose the whole fleet configuration.
// /deployments drift chip. A singleton so concurrent circuits share one composition gate (and a
// composition that started after they asked) instead of running N fleet snapshots at once.
// Constructed explicitly rather than by ctor scan: the compose seam is an optional parameter
// that only tests supply, and production must always get the real ConfigComposer default.
services.AddSingleton(sp => new Deployments.DeploymentDriftProbe(