fix(site-secrets): prune secret_mirror_status rows of decommissioned or renamed nodes

Each node now re-writes its mirror-status row every SecretMirrorStatusRefreshInterval
(10 min) as a heartbeat; SecretMirrorStatusReporter leaves off the health report and
deletes any row not refreshed for SecretMirrorStatusStaleAfter (1 h, validated >= 2x the
refresh). The delete is conditional on the checked_at it read, so a node that came back
keeps its fresh row; a returning node re-writes its row on its first pass.

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/DeployedSecretsMirror.cs

@@ -163,7 +163,10 @@ public sealed record MirrorResult(
 /// name; <c>SecretMirrorStatusReporter</c> on the active node reads every node's row onto the
 /// health report. Because every write replicates via CDC, the row is written only when the
 /// failure set (compared as sorted name+message pairs) differs from what this process last
-/// wrote — plus once on the first pass after start-up, so a restart refreshes a stale row. The
+/// wrote — plus once on the first pass after start-up, so a restart refreshes a stale row, and
+/// at least every <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/> so the
+/// row's <c>checked_at</c> works as a heartbeat: <c>SecretMirrorStatusReporter</c> prunes the
+/// row of a node that has stopped refreshing it (decommissioned or renamed). The
 /// write is best-effort: a failure logs a Warning (repeats at Debug) and is retried on the next
 /// pass, and never affects the pass's own <see cref="MirrorResult"/>.
 /// </para>
@@ -188,6 +191,9 @@ public sealed class DeployedSecretsMirror : BackgroundService
     /// </summary>
     private IReadOnlyList<SecretMirrorFailure>? _lastWrittenStatus;
 
+    /// <summary>When this process last wrote its <c>secret_mirror_status</c> row (heartbeat refresh).</summary>
+    private DateTimeOffset _lastWrittenStatusAt;
+
     /// <summary>Whether a failed status write has already been logged at Warning (de-spam).</summary>
     private bool _statusWriteFailureLogged;
 
@@ -358,7 +364,8 @@ public sealed class DeployedSecretsMirror : BackgroundService
 
     /// <summary>
     /// Writes this node's <c>secret_mirror_status</c> row when the pass's failure set differs
-    /// from the last one this process wrote (or nothing has been written yet since start-up).
+    /// from the last one this process wrote, nothing has been written yet since start-up, or the
+    /// last write is older than <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/>.
     /// Best-effort: never throws, so the pass result is unaffected by a status-write fault.
     /// See the class remarks (review finding M1).
     /// </summary>
@@ -370,15 +377,19 @@ public sealed class DeployedSecretsMirror : BackgroundService
             .ThenBy(f => f.Message, StringComparer.Ordinal)
             .ToList();
 
-        if (_lastWrittenStatus is not null && _lastWrittenStatus.SequenceEqual(current))
+        var now = _time.GetUtcNow();
+        if (_lastWrittenStatus is not null
+            && _lastWrittenStatus.SequenceEqual(current)
+            && now - _lastWrittenStatusAt < _options.Value.SecretMirrorStatusRefreshInterval)
         {
             return;
         }
 
         try
         {
-            await _storage.SetSecretMirrorStatusAsync(_nodeName, current, _time.GetUtcNow()).ConfigureAwait(false);
+            await _storage.SetSecretMirrorStatusAsync(_nodeName, current, now).ConfigureAwait(false);
             _lastWrittenStatus = current;
+            _lastWrittenStatusAt = now;
             _statusWriteFailureLogged = false;
         }
         catch (Exception ex)

src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Secrets/SecretMirrorStatusReporter.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
 using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
 using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
 
@@ -25,6 +26,15 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Secrets;
 /// first probe, fixed cadence, exceptions logged and swallowed. A failed read leaves the
 /// previous snapshot on the collector rather than clearing it.
 /// </para>
+/// <para>
+/// <b>Stale rows.</b> Every live node re-writes its row at least every
+/// <see cref="SiteRuntimeOptions.SecretMirrorStatusRefreshInterval"/>, so a row whose
+/// <c>checked_at</c> is older than <see cref="SiteRuntimeOptions.SecretMirrorStatusStaleAfter"/>
+/// belongs to a node that is no longer running its mirror — decommissioned, renamed (the row is
+/// keyed by node name), or down for that long. Such a row is left off the report and deleted
+/// (only while it still carries the stale <c>checked_at</c>, so a node that came back in the
+/// meantime keeps its fresh row); a returning node writes its row again on its first pass.
+/// </para>
 /// </remarks>
 public sealed class SecretMirrorStatusReporter : BackgroundService
 {
@@ -35,22 +45,33 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
     private readonly SiteStorageService _storage;
     private readonly ILogger<SecretMirrorStatusReporter> _logger;
     private readonly TimeSpan _pollInterval;
+    private readonly TimeSpan _staleAfter;
+    private readonly TimeProvider _time;
 
     /// <summary>Initializes a new instance of <see cref="SecretMirrorStatusReporter"/>.</summary>
     /// <param name="collector">The site health collector receiving the statuses.</param>
     /// <param name="storage">Site storage exposing <c>secret_mirror_status</c>.</param>
     /// <param name="logger">Logger instance.</param>
     /// <param name="pollInterval">Poll interval override; defaults to <see cref="DefaultPollInterval"/> (30 s).</param>
+    /// <param name="staleAfter">
+    /// Age after which a row is pruned; defaults to the
+    /// <see cref="SiteRuntimeOptions.SecretMirrorStatusStaleAfter"/> default (1 hour).
+    /// </param>
+    /// <param name="time">Clock used to age rows; defaults to <see cref="TimeProvider.System"/>.</param>
     public SecretMirrorStatusReporter(
         ISiteHealthCollector collector,
         SiteStorageService storage,
         ILogger<SecretMirrorStatusReporter> logger,
-        TimeSpan? pollInterval = null)
+        TimeSpan? pollInterval = null,
+        TimeSpan? staleAfter = null,
+        TimeProvider? time = null)
     {
         _collector = collector ?? throw new ArgumentNullException(nameof(collector));
         _storage = storage ?? throw new ArgumentNullException(nameof(storage));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         _pollInterval = pollInterval ?? DefaultPollInterval;
+        _staleAfter = staleAfter ?? new SiteRuntimeOptions().SecretMirrorStatusStaleAfter;
+        _time = time ?? TimeProvider.System;
     }
 
     /// <inheritdoc />
@@ -74,7 +95,8 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
     }
 
     /// <summary>
-    /// Reads every node's mirror status and pushes it onto the collector; never throws.
+    /// Reads every node's mirror status, prunes rows older than the stale threshold, and pushes
+    /// the rest onto the collector; never throws.
     /// Public so a test can drive one deterministic probe.
     /// </summary>
     /// <returns>A task that completes when the probe has finished.</returns>
@@ -83,7 +105,20 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
         try
         {
             var statuses = await _storage.GetSecretMirrorStatusesAsync().ConfigureAwait(false);
-            _collector.SetSecretMirrorStatuses(statuses);
+            var cutoff = _time.GetUtcNow() - _staleAfter;
+            var live = new List<SecretMirrorNodeStatus>(statuses.Count);
+            foreach (var status in statuses)
+            {
+                if (status.CheckedAtUtc >= cutoff)
+                {
+                    live.Add(status);
+                    continue;
+                }
+
+                await PruneAsync(status).ConfigureAwait(false);
+            }
+
+            _collector.SetSecretMirrorStatuses(live);
         }
         catch (Exception ex)
         {
@@ -92,4 +127,30 @@ public sealed class SecretMirrorStatusReporter : BackgroundService
             _logger.LogWarning(ex, "SecretMirrorStatusReporter probe failed; next tick will retry.");
         }
     }
+
+    /// <summary>
+    /// Deletes one stale row; a failed delete is logged and retried on the next probe (the row
+    /// stays off the report either way).
+    /// </summary>
+    private async Task PruneAsync(SecretMirrorNodeStatus status)
+    {
+        try
+        {
+            if (await _storage.DeleteSecretMirrorStatusIfUnchangedAsync(status.NodeName, status.CheckedAtUtc)
+                    .ConfigureAwait(false))
+            {
+                _logger.LogInformation(
+                    "Removed the secret mirror status row for node {NodeName}: not refreshed since {CheckedAtUtc:O} " +
+                    "(older than {StaleAfter}); the node is decommissioned, renamed or down. It is re-created " +
+                    "on the node's next mirror pass if it returns.",
+                    status.NodeName, status.CheckedAtUtc, _staleAfter);
+            }
+        }
+        catch (Exception ex)
+        {
+            _logger.LogWarning(ex,
+                "Failed to remove the stale secret mirror status row for node {NodeName}; next tick will retry.",
+                status.NodeName);
+        }
+    }
 }