fix(ops): wonder site config gains required audit DB path; explicit LocalDb read-page cap; rate-limited observer drop logging

F1: deploy/wonder-app-vd03/appsettings.Site.json (outside git, WP1.2's StartupValidator gate applies live on next install/upgrade) was missing the now-required AuditLog:SiteWriter:DatabasePath, added pointing at E:\ApiInstall\ScadaBridge\site\data\auditlog.db alongside the file's existing SiteEventLog/LocalDb paths; scanned deploy/ for other Site-role appsettings with the same gap (none) and confirmed wonder does not pin LocalDb:Replication:MaxBatchSize (F2 doesn't apply there). F2: re-pin an explicit LocalDb:Replication:MaxBatchSize=64 on docker/site-a node-a and node-b. MaxBatchBytes (2 MB default) only bounds the wire message via the per-message split in SyncSession.PumpLoopAsync; MaxBatchSize separately bounds the DB read page in OplogStore.ReadBatchAboveAsync/SnapshotStreamer, which materializes the whole page into memory before that split runs. Left at the 500 default, a reconnect drain of worst-case config_json rows could transiently allocate ~35 MB per read even though every wire message stayed within budget. Updated the CLAUDE.md LocalDb bullet to stop implying the row cap is fully redundant with the byte budget (topology-guide.md has no matching claim). F3: StoreAndForwardService's observer-queue onDropped callback logged a Warning per dropped item, flooding logs at sweep rate for a stuck observer with a large queue. LogObserverQueueDrop now logs once immediately on the first drop of an episode, then throttles to at most one rollup Warning per minute while drops continue, reporting the count dropped since the last log; the cumulative ObserverQueueDroppedCount counter is unaffected. Extended StoreAndForwardServiceTests with ObserverQueue_ManyDropsInOneEpisode_LogsExactlyOneWarning, which floods the bounded queue and pins exactly one drop-related Warning log for the episode via a small CapturingLogger test double. dotnet build ZB.MOM.WW.ScadaBridge.slnx: 0 warnings, 0 errors. dotnet test StoreAndForward.Tests: 134/134 passed. dotnet test Host.Tests: 490/490 passed.
2026-08-14 23:31:52 -04:00
parent b1de9dfdd4
commit 56c99c92c3
5 changed files with 210 additions and 26 deletions
@@ -1,3 +1,4 @@
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
 using ZB.MOM.WW.ScadaBridge.Commons.Types;
@@ -965,4 +966,109 @@ public class StoreAndForwardServiceTests : IAsyncLifetime, IDisposable
            TestLocalDb.DeleteFiles(path);
        }
    }
+
+    /// <summary>
+    /// Captures every message logged through it, keyed by <see cref="LogLevel"/>. Minimal
+    /// test double — no scopes, no filtering — just enough to assert on log VOLUME.
+    /// </summary>
+    private sealed class CapturingLogger<T> : ILogger<T>
+    {
+        private readonly List<(LogLevel Level, string Message)> _entries = new();
+        private readonly object _gate = new();
+
+        public IReadOnlyList<(LogLevel Level, string Message)> Entries
+        {
+            get { lock (_gate) return _entries.ToList(); }
+        }
+
+        public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
+
+        public bool IsEnabled(LogLevel logLevel) => true;
+
+        public void Log<TState>(
+            LogLevel logLevel, EventId eventId, TState state, Exception? exception,
+            Func<TState, Exception?, string> formatter)
+        {
+            var message = formatter(state, exception);
+            lock (_gate) _entries.Add((logLevel, message));
+        }
+    }
+
+    /// <summary>
+    /// arch-review adversarial finding F3: the observer-queue onDropped callback used to
+    /// log a Warning PER dropped item — a stuck observer with a large
+    /// <see cref="StoreAndForwardOptions.ObserverQueueCapacity"/> would flood the log at
+    /// sweep rate. Extends <see cref="ObserverQueue_BoundedCapacity_DropsOldestAndCountsDrops"/>
+    /// (same bounded-queue setup) to pin the fix: many drops in one episode still
+    /// increment <see cref="StoreAndForwardService.ObserverQueueDroppedCount"/> per drop, but
+    /// produce exactly ONE observer-queue-drop Warning log — the first-drop Warning — because
+    /// the episode never runs long enough to cross the periodic rollup interval.
+    /// </summary>
+    [Fact]
+    public async Task ObserverQueue_ManyDropsInOneEpisode_LogsExactlyOneWarning()
+    {
+        var gate = new TaskCompletionSource();
+        var observer = new BlockingObserver(gate);
+        var logger = new CapturingLogger<StoreAndForwardService>();
+
+        var localDb = TestLocalDb.CreateTemp("ObsQueueDropLogRollup");
+        var storage = new StoreAndForwardStorage(localDb.Db, NullLogger<StoreAndForwardStorage>.Instance);
+        await storage.InitializeAsync();
+        var service = new StoreAndForwardService(
+            storage,
+            new StoreAndForwardOptions
+            {
+                DefaultRetryInterval = TimeSpan.Zero,
+                DefaultMaxRetries = 5,
+                RetryTimerInterval = TimeSpan.FromHours(1), // timer never fires in-test
+                ObserverQueueCapacity = 2,
+            },
+            logger,
+            cachedCallObserver: observer,
+            siteId: "site-78");
+
+        await service.StartAsync();
+        try
+        {
+            service.RegisterDeliveryHandler(StoreAndForwardCategory.ExternalSystem,
+                _ => throw new HttpRequestException("transient"));
+
+            // Enqueue far more than the bounded capacity (2) so the queue overflows many
+            // times over in one sweep — same mechanism as the sibling test above, just a
+            // larger flood to make a per-item log flood obvious if the fix regresses.
+            for (var i = 0; i < 50; i++)
+            {
+                await service.EnqueueAsync(
+                    StoreAndForwardCategory.ExternalSystem, $"t{i}", "{}",
+                    attemptImmediateDelivery: false, retryInterval: TimeSpan.Zero,
+                    messageId: TrackedOperationId.New().ToString());
+            }
+
+            await service.RetryPendingMessagesAsync();
+
+            var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(5);
+            while (DateTime.UtcNow < deadline && service.ObserverQueueDroppedCount < 10)
+                await Task.Delay(10);
+
+            Assert.True(service.ObserverQueueDroppedCount >= 10,
+                "expected many notifications to be dropped once the bounded queue filled");
+
+            var dropWarnings = logger.Entries
+                .Where(e => e.Level == LogLevel.Warning
+                    && e.Message.Contains("audit-observer queue", StringComparison.Ordinal))
+                .ToList();
+
+            Assert.True(dropWarnings.Count == 1,
+                $"expected exactly one observer-queue-drop Warning per episode (rate-limited), " +
+                $"but got {dropWarnings.Count} for {service.ObserverQueueDroppedCount} drops");
+        }
+        finally
+        {
+            gate.TrySetResult();
+            await service.StopAsync();
+            var path = localDb.Path;
+            localDb.Dispose();
+            TestLocalDb.DeleteFiles(path);
+        }
+    }
 }