dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(sitelog): sampled per-run events; interval run summaries; site_events replication policy pinned

Browse Source 
Implements WP3.2 stage (b) per docs/plans/2026-08-15-site-events-policy-design.md.

- Per-run instance-script Started/Completed Info site events are now off by
  default (SiteRuntimeOptions.PerRunScriptEvents=false) instead of firing on
  every run, closing the dominant site_events writer. Gated at the ScriptRunLauncher
  call sites (moved there from ScriptExecutionActor by WP3.1). Error-level events
  (timeout/failure/stuck-watchdog/recursion-limit) remain unconditional.
- ScriptRunSummaryRecorder accumulates per-(instance, script) run counters and a
  new site-only ScriptRunSummaryFlushService emits one aggregate "script" Info
  site event per ScriptRunSummaryIntervalSeconds (default 300s), top-50-script
  breakdown with an "others" rollup, zero-activity intervals emit nothing.
- Per-script opt-in via PerRunScriptEventScripts ("Instance/Script" exact or
  "Instance/*" wildcard), matched by the new pure ScriptRunEventPolicy. All three
  options are read from IOptionsMonitor<SiteRuntimeOptions> per run, so the
  policy is hot-togglable without a restart.
- Fixed the stale "event log is not replicated" comment at AkkaHostedService.cs
  (~905): site_events IS registered in SiteLocalDbSetup.ReplicatedTables — the
  singleton is what makes queries always hit the actively-written copy;
  replication is what gives the singleton history to read after a failover
  (memo Decision (b)). site_events replication itself is unchanged (still
  registered) and already pinned by
  tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbCdcRegistrationTests.cs.
- Updated Component-SiteEventLogging.md (Volume Policy section, corrected
  Storage/replication rationale) and Component-SiteRuntime.md (Script Run
  Launch + Error Handling sections).
This commit is contained in:
Joseph Doherty
2026-08-14 22:56:08 -04:00
parent 799fd041ec
commit c254d0740e
15 changed files with 922 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRunLauncher.cs
+44 -12
View File
@@ -1,7 +1,9 @@
using System.Diagnostics;
using Akka.Actor;
using Microsoft.CodeAnalysis.Scripting;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
@@ -221,6 +223,20 @@ internal static class ScriptRunLauncher
// it is available to the catch blocks regardless of scope state.
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
// WP3.2 (site_events volume policy): both are singletons, resolved from the root
// provider up front so they too are available to the catch blocks. optionsMonitor is
// null in tests that pass no serviceProvider (SingleServiceProvider et al.) — the
// constructor-passed `options` is the fallback, exactly like siteEventLogger's null
// path degrades to "no site event log". Reading CurrentValue HERE (once per run, at
// the top) rather than re-reading it at each call site is what "hot-togglable" means
// in practice: an operator's live edit to PerRunScriptEvents/PerRunScriptEventScripts
// takes effect on the NEXT run, without a restart.
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
var optionsMonitor = serviceProvider?.GetService<IOptionsMonitor<SiteRuntimeOptions>>();
var effectiveOptions = optionsMonitor?.CurrentValue ?? options;
var emitPerRunEvents = ScriptRunEventPolicy.ShouldEmitPerRun(effectiveOptions, instanceName, scriptName);
var runStopwatch = Stopwatch.StartNew();
// WP3.1: identify the worker this body is holding, so the watchdog can detach and
// replace it if the body wedges. Read in the FIRST synchronous segment — a script
// that later hops threads on an await no longer holds this worker, and the run-stamp
@@ -350,12 +366,19 @@ internal static class ScriptRunLauncher
Scope = scope
};
// Operational `script` event — execution started. Fire-and-forget
// (the `_ =` discards the task) so the event log can never block or
// fault the script's own run; mirrors the existing Error-path emit.
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' started");
// WP3.2: the aggregate ALWAYS records this run (that's the point of the interval
// summary), but the per-run Info event itself is gated by ScriptRunEventPolicy —
// off by default, on globally via PerRunScriptEvents, or on for named scripts via
// PerRunScriptEventScripts. Fire-and-forget (the `_ =` discards the task) so the
// event log can never block or fault the script's own run; mirrors the existing
// Error-path emit, which stays unconditional regardless of this policy.
summaryRecorder?.RecordStarted(instanceName, scriptName);
if (emitPerRunEvents)
{
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' started");
}
var state = await compiledScript.RunAsync(globals, cts.Token);
@@ -365,10 +388,15 @@ internal static class ScriptRunLauncher
replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
}
// Operational `script` event — execution completed successfully.
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' completed");
// Operational `script` event — execution completed successfully. Same gating as
// the started event above.
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
if (emitPerRunEvents)
{
_ = siteEventLogger?.LogEventAsync(
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
$"Script '{scriptName}' on instance '{instanceName}' completed");
}
// Notify the owning ScriptActor of completion (also releases its in-flight slot).
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
@@ -376,10 +404,12 @@ internal static class ScriptRunLauncher
catch (OperationCanceledException)
{
healthCollector?.IncrementScriptError();
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);
// Failures recorded to site event log; script NOT disabled after failure.
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
_ = siteEventLogger?.LogEventAsync(
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
@@ -393,7 +423,9 @@ internal static class ScriptRunLauncher
catch (Exception ex)
{
healthCollector?.IncrementScriptError();
// Failures recorded to site event log; script NOT disabled after failure.
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
scriptName, instanceName, runId);