dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(sitelog): sampled per-run events; interval run summaries; site_events replication policy pinned

Browse Source 
Implements WP3.2 stage (b) per docs/plans/2026-08-15-site-events-policy-design.md.

- Per-run instance-script Started/Completed Info site events are now off by
  default (SiteRuntimeOptions.PerRunScriptEvents=false) instead of firing on
  every run, closing the dominant site_events writer. Gated at the ScriptRunLauncher
  call sites (moved there from ScriptExecutionActor by WP3.1). Error-level events
  (timeout/failure/stuck-watchdog/recursion-limit) remain unconditional.
- ScriptRunSummaryRecorder accumulates per-(instance, script) run counters and a
  new site-only ScriptRunSummaryFlushService emits one aggregate "script" Info
  site event per ScriptRunSummaryIntervalSeconds (default 300s), top-50-script
  breakdown with an "others" rollup, zero-activity intervals emit nothing.
- Per-script opt-in via PerRunScriptEventScripts ("Instance/Script" exact or
  "Instance/*" wildcard), matched by the new pure ScriptRunEventPolicy. All three
  options are read from IOptionsMonitor<SiteRuntimeOptions> per run, so the
  policy is hot-togglable without a restart.
- Fixed the stale "event log is not replicated" comment at AkkaHostedService.cs
  (~905): site_events IS registered in SiteLocalDbSetup.ReplicatedTables — the
  singleton is what makes queries always hit the actively-written copy;
  replication is what gives the singleton history to read after a failover
  (memo Decision (b)). site_events replication itself is unchanged (still
  registered) and already pinned by
  tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbCdcRegistrationTests.cs.
- Updated Component-SiteEventLogging.md (Volume Policy section, corrected
  Storage/replication rationale) and Component-SiteRuntime.md (Script Run
  Launch + Error Handling sections).
This commit is contained in:
Joseph Doherty
2026-08-14 22:56:08 -04:00
parent 799fd041ec
commit c254d0740e
15 changed files with 922 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests/Actors/ScriptRunLauncherParityTests.cs
+108 -5
View File
@@ -64,12 +64,16 @@ public class ScriptRunLauncherParityTests : TestKit, IDisposable
return script;
}
private static SiteRuntimeOptions Options(int timeoutSeconds = 30, int graceMs = 30000)
private static SiteRuntimeOptions Options(
int timeoutSeconds = 30, int graceMs = 30000, bool perRunScriptEvents = false)
=> new()
{
MaxScriptCallDepth = 10,
ScriptExecutionTimeoutSeconds = timeoutSeconds,
StuckScriptGraceMs = graceMs
StuckScriptGraceMs = graceMs,
// WP3.2: per-run Started/Completed Info events default OFF; tests that pin the
// legacy per-run event shape opt back in explicitly via this flag.
PerRunScriptEvents = perRunScriptEvents
};
private static ResolvedScript CallScript(int? timeoutSeconds = null) => new()
@@ -382,8 +386,10 @@ public class ScriptRunLauncherParityTests : TestKit, IDisposable
public void SuccessfulRun_EmitsStartedThenCompletedInfoEvents()
{
var siteLog = new FakeSiteEventLogger();
// WP3.2: per-run Info events are opt-in now (default off, interval summary instead) —
// this test pins the legacy per-run shape, so it opts back in explicitly.
var actor = BuildScriptActor(
CompileScript("return 7 * 6;"), Options(), new SingleServiceProvider(siteLog));
CompileScript("return 7 * 6;"), Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt"), caller.Ref);
@@ -408,9 +414,11 @@ public class ScriptRunLauncherParityTests : TestKit, IDisposable
public void FailingRun_EmitsStartedInfoThenErrorEvent()
{
var siteLog = new FakeSiteEventLogger();
// WP3.2: the started Info event is opt-in (default off); the Error event on failure
// stays unconditional regardless. This test pins the opted-in shape explicitly.
var actor = BuildScriptActor(
CompileScript("throw new InvalidOperationException(\"boom\");"),
Options(), new SingleServiceProvider(siteLog));
Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-evt-err"), caller.Ref);
@@ -430,8 +438,11 @@ public class ScriptRunLauncherParityTests : TestKit, IDisposable
public void FireAndForgetRun_NeedsNoReplyTarget()
{
var siteLog = new FakeSiteEventLogger();
// WP3.2: opted in so the completed Info event (the observable proof the
// fire-and-forget run actually ran to completion) is emitted; the policy itself is
// covered separately by ScriptRunEventPolicyTests.
var actor = BuildScriptActor(
CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog));
CompileScript("return 1;"), Options(perRunScriptEvents: true), new SingleServiceProvider(siteLog));
// Trigger-driven spawns pass ActorRefs.NoSender as replyTo; drive that path via an
// interval-free Call script by telling the actor to run with no sender.
@@ -442,6 +453,98 @@ public class ScriptRunLauncherParityTests : TestKit, IDisposable
r => r.Message.Contains("completed", StringComparison.OrdinalIgnoreCase)),
TimeSpan.FromSeconds(10));
}
// ── WP3.2: site_events volume policy — actor-level integration ────────────────
//
// The pure ScriptRunEventPolicy matching rules (exact/wildcard/global/non-matching) are
// unit-tested directly in ScriptRunEventPolicyTests; these prove the policy is actually
// wired through the real ScriptActor -> ScriptRunLauncher path: the default is silent but
// the aggregate still sees the run, per-script opt-in works end to end, and the
// IOptionsMonitor read makes the switch hot-togglable without an actor restart.
[Fact]
public async Task DefaultOptions_SuccessfulRun_EmitsNoPerRunEvents_ButTheSummaryRecorderSeesIt()
{
var siteLog = new FakeSiteEventLogger();
var recorder = new ScriptRunSummaryRecorder();
var actor = BuildScriptActor(
CompileScript("return 1;"), Options(),
new SingleServiceProvider(siteLog, summaryRecorder: recorder));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-default-off"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
// RecordCompleted happens-before the ScriptExecutionCompleted Tell that releases the
// in-flight slot (see RunScriptAsync), so waiting on RunsInFlight == 0 makes the
// subsequent flush deterministic — no sleep, no retry-flush race.
AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5));
Assert.Empty(siteLog.OfType("script"));
var flushLog = new FakeSiteEventLogger();
var emitted = await recorder.FlushAsync(flushLog);
Assert.True(emitted);
Assert.Equal(
"1 runs: 1 completed, 0 failed, 0 timed out across 1 scripts",
flushLog.OfType("script").Single().Message);
}
[Fact]
public void PerScriptOptIn_ExactMatch_RestoresPerRunEventsForThatScriptOnly()
{
var siteLog = new FakeSiteEventLogger();
var options = Options();
options.PerRunScriptEventScripts = ["Inst1/Runner"];
var actor = BuildScriptActor(
CompileScript("return 1;"), options, new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-exact-optin"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5));
}
[Fact]
public void PerScriptOptIn_InstanceWildcard_RestoresPerRunEvents()
{
var siteLog = new FakeSiteEventLogger();
var options = Options();
options.PerRunScriptEventScripts = ["Inst1/*"];
var actor = BuildScriptActor(
CompileScript("return 1;"), options, new SingleServiceProvider(siteLog));
var caller = CreateTestProbe();
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-wildcard-optin"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5));
}
[Fact]
public void HotToggle_FlippingTheOptionsMonitor_ChangesTheNextRunWithNoActorRestart()
{
var siteLog = new FakeSiteEventLogger();
var monitor = new TestOptionsMonitor<SiteRuntimeOptions>(Options());
var actor = BuildScriptActor(
CompileScript("return 1;"), Options(), new SingleServiceProvider(siteLog, monitor));
var caller = CreateTestProbe();
// Off: the monitor's initial value has PerRunScriptEvents=false.
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-hot-1"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() => Assert.Equal(0, actor.UnderlyingActor.RunsInFlight), TimeSpan.FromSeconds(5));
Assert.Empty(siteLog.OfType("script"));
// Flip live — same actor, no restart, no redeploy.
monitor.Set(Options(perRunScriptEvents: true));
actor.Tell(new ScriptCallRequest("Runner", null, 0, "corr-hot-2"), caller.Ref);
caller.ExpectMsg<ScriptCallResult>(TimeSpan.FromSeconds(10));
AwaitAssert(() => Assert.Equal(2, siteLog.OfType("script").Count), TimeSpan.FromSeconds(5));
}
}
/// <summary>