fix(site-secrets): keep a live node's mirror-status row from losing a race with the peer's prune

Rig finding on the upgrade restart: both nodes' rows were older than the stale threshold
(the old build never refreshed them), so node-a's reporter pruned node-b's row while node-b
wrote a fresh one, and the replicated delete won. The reporter now prunes nothing during its
first refresh interval after start-up, and each mirror pass re-writes its own row if it no
longer holds what the mirror last wrote.

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Persistence/SiteStorageService.cs

@@ -1006,6 +1006,26 @@ public class SiteStorageService
         return results;
     }
 
+    /// <summary>
+    /// Returns the raw <c>checked_at</c> of <paramref name="nodeName"/>'s
+    /// <c>secret_mirror_status</c> row, or <see langword="null"/> when the node has no row — a
+    /// cheap local read the mirror uses to notice its row was deleted by the peer's prune.
+    /// </summary>
+    /// <param name="nodeName">The row key.</param>
+    /// <returns>The stored <c>checked_at</c> text, or <see langword="null"/>.</returns>
+    public async Task<string?> GetSecretMirrorStatusCheckedAtAsync(string nodeName)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(nodeName);
+
+        await using var connection = OpenConnection();
+
+        await using var command = connection.CreateCommand();
+        command.CommandText = "SELECT checked_at FROM secret_mirror_status WHERE node_name = @nodeName";
+        command.Parameters.AddWithValue("@nodeName", nodeName);
+
+        return await command.ExecuteScalarAsync() as string;
+    }
+
     /// <summary>
     /// Deletes <paramref name="nodeName"/>'s <c>secret_mirror_status</c> row, but only while it
     /// still carries <paramref name="checkedAtUtc"/> — so a row the node re-wrote after the caller

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs

@@ -166,7 +166,10 @@ public sealed record MirrorResult(
 /// wrote — plus once on the first pass after start-up, so a restart refreshes a stale row, and
 /// at least every <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/> so the
 /// row's <c>checked_at</c> works as a heartbeat: <c>SecretMirrorStatusReporter</c> prunes the
-/// row of a node that has stopped refreshing it (decommissioned or renamed). The
+/// row of a node that has stopped refreshing it (decommissioned or renamed). Each pass also
+/// checks (a local read) that the row still holds what this process last wrote, and re-writes it
+/// if not — a peer's prune racing this node's own write can replicate a delete over it, e.g. when
+/// both nodes restart together with rows older than the stale threshold. The
 /// write is best-effort: a failure logs a Warning (repeats at Debug) and is retried on the next
 /// pass, and never affects the pass's own <see cref="MirrorResult"/>.
 /// </para>
@@ -365,7 +368,9 @@ public sealed class DeployedSecretsMirror : BackgroundService
     /// <summary>
     /// Writes this node's <c>secret_mirror_status</c> row when the pass's failure set differs
     /// from the last one this process wrote, nothing has been written yet since start-up, or the
-    /// last write is older than <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/>.
+    /// last write is older than <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/>,
+    /// or the stored row no longer holds what this process wrote (deleted or overwritten by a
+    /// replicated change).
     /// Best-effort: never throws, so the pass result is unaffected by a status-write fault.
     /// See the class remarks (review finding M1).
     /// </summary>
@@ -378,15 +383,18 @@ public sealed class DeployedSecretsMirror : BackgroundService
             .ToList();
 
         var now = _time.GetUtcNow();
-        if (_lastWrittenStatus is not null
-            && _lastWrittenStatus.SequenceEqual(current)
-            && now - _lastWrittenStatusAt < _options.Value.SecretMirrorStatusRefreshInterval)
-        {
-            return;
-        }
 
         try
         {
+            if (_lastWrittenStatus is not null
+                && _lastWrittenStatus.SequenceEqual(current)
+                && now - _lastWrittenStatusAt < _options.Value.SecretMirrorStatusRefreshInterval
+                && await _storage.GetSecretMirrorStatusCheckedAtAsync(_nodeName).ConfigureAwait(false)
+                    == _lastWrittenStatusAt.ToUniversalTime().ToString("O"))
+            {
+                return;
+            }
+
             await _storage.SetSecretMirrorStatusAsync(_nodeName, current, now).ConfigureAwait(false);
             _lastWrittenStatus = current;
             _lastWrittenStatusAt = now;

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/SecretMirrorStatusReporter.cs

@@ -34,6 +34,9 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
 /// keyed by node name), or down for that long. Such a row is left off the report and deleted
 /// (only while it still carries the stale <c>checked_at</c>, so a node that came back in the
 /// meantime keeps its fresh row); a returning node writes its row again on its first pass.
+/// Nothing is pruned during the reporter's first <c>pruneGrace</c> (the refresh interval) after
+/// start-up: when both nodes of a pair restart together, every live node's row can still be old
+/// until its mirror's first write has replicated, and pruning it then only races that write.
 /// </para>
 /// </remarks>
 public sealed class SecretMirrorStatusReporter : BackgroundService
@@ -47,6 +50,7 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
     private readonly TimeSpan _pollInterval;
     private readonly TimeSpan _staleAfter;
     private readonly TimeProvider _time;
+    private readonly DateTimeOffset _pruneFrom;
 
     /// <summary>Initializes a new instance of <see cref="SecretMirrorStatusReporter"/>.</summary>
     /// <param name="collector">The site health collector receiving the statuses.</param>
@@ -58,13 +62,18 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
     /// <see cref="SiteRuntimeOptions.SecretMirrorStatusStaleAfter"/> default (1 hour).
     /// </param>
     /// <param name="time">Clock used to age rows; defaults to <see cref="TimeProvider.System"/>.</param>
+    /// <param name="pruneGrace">
+    /// How long after construction nothing is pruned; defaults to zero. Under DI it is the
+    /// <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/>.
+    /// </param>
     public SecretMirrorStatusReporter(
         ISiteHealthCollector collector,
         SiteStorageService storage,
         ILogger<SecretMirrorStatusReporter> logger,
         TimeSpan? pollInterval = null,
         TimeSpan? staleAfter = null,
-        TimeProvider? time = null)
+        TimeProvider? time = null,
+        TimeSpan? pruneGrace = null)
     {
         _collector = collector ?? throw new ArgumentNullException(nameof(collector));
         _storage = storage ?? throw new ArgumentNullException(nameof(storage));
@@ -72,6 +81,7 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
         _pollInterval = pollInterval ?? DefaultPollInterval;
         _staleAfter = staleAfter ?? new SiteRuntimeOptions().SecretMirrorStatusStaleAfter;
         _time = time ?? TimeProvider.System;
+        _pruneFrom = _time.GetUtcNow() + (pruneGrace ?? TimeSpan.Zero);
     }
 
     /// <inheritdoc />
@@ -105,11 +115,12 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
         try
         {
             var statuses = await _storage.GetSecretMirrorStatusesAsync().ConfigureAwait(false);
-            var cutoff = _time.GetUtcNow() - _staleAfter;
+            var now = _time.GetUtcNow();
+            var cutoff = now - _staleAfter;
             var live = new List<SecretMirrorNodeStatus>(statuses.Count);
             foreach (var status in statuses)
             {
-                if (status.CheckedAtUtc >= cutoff)
+                if (status.CheckedAtUtc >= cutoff || now < _pruneFrom)
                 {
                     live.Add(status);
                     continue;

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/ServiceCollectionExtensions.cs

@@ -147,7 +147,8 @@ public static class ServiceCollectionExtensions
             sp.GetRequiredService<SiteStorageService>(),
             sp.GetRequiredService<ILogger<SecretMirrorStatusReporter>>(),
             staleAfter: sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value.SecretMirrorStatusStaleAfter,
-            time: sp.GetRequiredService<TimeProvider>()));
+            time: sp.GetRequiredService<TimeProvider>(),
+            pruneGrace: sp.GetRequiredService<IOptions<SiteRuntimeOptions>>().Value.SecretMirrorStatusRefreshInterval));
 
         return services;
     }