perf(sitelog): sampled per-run events; interval run summaries; site_events replication policy pinned

Implements WP3.2 stage (b) per docs/plans/2026-08-15-site-events-policy-design.md. - Per-run instance-script Started/Completed Info site events are now off by default (SiteRuntimeOptions.PerRunScriptEvents=false) instead of firing on every run, closing the dominant site_events writer. Gated at the ScriptRunLauncher call sites (moved there from ScriptExecutionActor by WP3.1). Error-level events (timeout/failure/stuck-watchdog/recursion-limit) remain unconditional. - ScriptRunSummaryRecorder accumulates per-(instance, script) run counters and a new site-only ScriptRunSummaryFlushService emits one aggregate "script" Info site event per ScriptRunSummaryIntervalSeconds (default 300s), top-50-script breakdown with an "others" rollup, zero-activity intervals emit nothing. - Per-script opt-in via PerRunScriptEventScripts ("Instance/Script" exact or "Instance/*" wildcard), matched by the new pure ScriptRunEventPolicy. All three options are read from IOptionsMonitor<SiteRuntimeOptions> per run, so the policy is hot-togglable without a restart. - Fixed the stale "event log is not replicated" comment at AkkaHostedService.cs (~905): site_events IS registered in SiteLocalDbSetup.ReplicatedTables — the singleton is what makes queries always hit the actively-written copy; replication is what gives the singleton history to read after a failover (memo Decision (b)). site_events replication itself is unchanged (still registered) and already pinned by tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbCdcRegistrationTests.cs. - Updated Component-SiteEventLogging.md (Volume Policy section, corrected Storage/replication rationale) and Component-SiteRuntime.md (Script Run Launch + Error Handling sections).
2026-08-14 22:56:08 -04:00
parent 799fd041ec
commit c254d0740e
15 changed files with 922 additions and 27 deletions
@@ -0,0 +1,169 @@
+using System.Globalization;
+using System.Text.Json;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
+using ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.TestSupport;
+
+namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests.Scripts;
+
+/// <summary>
+/// WP3.2 (site_events volume policy — docs/plans/2026-08-15-site-events-policy-design.md,
+/// stage (b) test plan item 5): <see cref="ScriptRunSummaryRecorder"/> in isolation — no
+/// actor, no hosted service, just the accumulate/flush contract.
+/// </summary>
+public class ScriptRunSummaryRecorderTests
+{
+    [Fact]
+    public async Task ZeroActivity_FlushEmitsNoRow()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+        var siteLog = new FakeSiteEventLogger();
+
+        var emitted = await recorder.FlushAsync(siteLog);
+
+        Assert.False(emitted);
+        Assert.Empty(siteLog.Entries);
+    }
+
+    [Fact]
+    public async Task MixedOutcomes_FlushEmitsOneRowWithCorrectTotals()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+
+        // Two scripts, mixed outcomes.
+        recorder.RecordStarted("Inst1", "A");
+        recorder.RecordCompleted("Inst1", "A", 10);
+        recorder.RecordStarted("Inst1", "A");
+        recorder.RecordCompleted("Inst1", "A", 30);
+
+        recorder.RecordStarted("Inst1", "B");
+        recorder.RecordFailed("Inst1", "B", 5);
+
+        recorder.RecordTimedOut("Inst2", "C", 100);
+
+        var siteLog = new FakeSiteEventLogger();
+        var emitted = await recorder.FlushAsync(siteLog);
+
+        Assert.True(emitted);
+        var rows = siteLog.OfType("script");
+        Assert.Single(rows);
+
+        var row = rows[0];
+        Assert.Equal("Info", row.Severity);
+        Assert.Null(row.InstanceId);
+        Assert.Equal("ScriptRunSummary", row.Source);
+
+        // Started: A x2 + B x1 = 3 (C never reaches RecordStarted — it is shed at dequeue,
+        // which is exactly why ranking uses Started+TimedOut, not Started alone).
+        // Completed: A x2 = 2. Failed: B x1 = 1. TimedOut: C x1 = 1. Scripts touched: A, B, C.
+        Assert.Equal("3 runs: 2 completed, 1 failed, 1 timed out across 3 scripts", row.Message);
+
+        Assert.NotNull(row.Details);
+        using var doc = JsonDocument.Parse(row.Details!);
+        var scripts = doc.RootElement.GetProperty("scripts");
+        Assert.Equal(3, scripts.GetArrayLength());
+    }
+
+    [Fact]
+    public async Task CountersReset_AfterFlush()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+        recorder.RecordStarted("Inst1", "A");
+        recorder.RecordCompleted("Inst1", "A", 1);
+
+        var firstLog = new FakeSiteEventLogger();
+        Assert.True(await recorder.FlushAsync(firstLog));
+
+        // Nothing recorded since the first flush — the second flush must be idle.
+        var secondLog = new FakeSiteEventLogger();
+        var emitted = await recorder.FlushAsync(secondLog);
+
+        Assert.False(emitted);
+        Assert.Empty(secondLog.Entries);
+    }
+
+    [Fact]
+    public async Task MoreThanFiftyScripts_CapsBreakdownAndRollsUpTheRest()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+
+        // 60 distinct scripts, each with a distinct run count so ranking is deterministic:
+        // script "S00" gets 60 runs, "S01" gets 59, ... "S59" gets 1.
+        for (var i = 0; i < 60; i++)
+        {
+            var runs = 60 - i;
+            var scriptName = $"S{i:D2}";
+            for (var r = 0; r < runs; r++)
+            {
+                recorder.RecordStarted("Inst1", scriptName);
+                recorder.RecordCompleted("Inst1", scriptName, 1);
+            }
+        }
+
+        var siteLog = new FakeSiteEventLogger();
+        var emitted = await recorder.FlushAsync(siteLog);
+
+        Assert.True(emitted);
+        var row = siteLog.OfType("script").Single();
+        Assert.Contains("60 scripts", row.Message);
+
+        using var doc = JsonDocument.Parse(row.Details!);
+        var root = doc.RootElement;
+        var scripts = root.GetProperty("scripts");
+        Assert.Equal(ScriptRunSummaryRecorder.TopScriptCap, scripts.GetArrayLength());
+
+        var others = root.GetProperty("others");
+        Assert.Equal(10, others.GetProperty("scriptCount").GetInt64());
+        // The ten lowest-run scripts (S50..S59) have run counts 10 down to 1 => sum 55.
+        Assert.Equal(55, others.GetProperty("started").GetInt64());
+
+        // The top entry must be the highest-run script (S00, 60 runs), not an arbitrary one.
+        var top = scripts[0];
+        Assert.Equal("S00", top.GetProperty("scriptName").GetString());
+        Assert.Equal(60, top.GetProperty("started").GetInt64());
+    }
+
+    [Fact]
+    public async Task ConcurrentIncrements_AreRaceFree()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+        const int perTask = 500;
+        const int taskCount = 8;
+
+        var tasks = Enumerable.Range(0, taskCount).Select(_ => Task.Run(() =>
+        {
+            for (var i = 0; i < perTask; i++)
+            {
+                recorder.RecordStarted("Inst1", "Hot");
+                recorder.RecordCompleted("Inst1", "Hot", 1);
+            }
+        }));
+
+        await Task.WhenAll(tasks);
+
+        var siteLog = new FakeSiteEventLogger();
+        Assert.True(await recorder.FlushAsync(siteLog));
+
+        var row = siteLog.OfType("script").Single();
+        var expectedTotal = (perTask * taskCount).ToString("N0", CultureInfo.InvariantCulture);
+        Assert.Contains($"{expectedTotal} runs", row.Message);
+        Assert.Contains($"{expectedTotal} completed", row.Message);
+    }
+
+    [Fact]
+    public async Task DurationTracking_ReportsAverageAndMax()
+    {
+        var recorder = new ScriptRunSummaryRecorder();
+        recorder.RecordStarted("Inst1", "A");
+        recorder.RecordCompleted("Inst1", "A", 10);
+        recorder.RecordStarted("Inst1", "A");
+        recorder.RecordCompleted("Inst1", "A", 30);
+
+        var siteLog = new FakeSiteEventLogger();
+        await recorder.FlushAsync(siteLog);
+
+        using var doc = JsonDocument.Parse(siteLog.OfType("script").Single().Details!);
+        var script = doc.RootElement.GetProperty("scripts")[0];
+        Assert.Equal(20, script.GetProperty("avgDurationMs").GetInt64());
+        Assert.Equal(30, script.GetProperty("maxDurationMs").GetInt64());
+    }
+}