perf(sitelog): sampled per-run events; interval run summaries; site_events replication policy pinned
Implements WP3.2 stage (b) per docs/plans/2026-08-15-site-events-policy-design.md.
- Per-run instance-script Started/Completed Info site events are now off by
default (SiteRuntimeOptions.PerRunScriptEvents=false) instead of firing on
every run, closing the dominant site_events writer. Gated at the ScriptRunLauncher
call sites (moved there from ScriptExecutionActor by WP3.1). Error-level events
(timeout/failure/stuck-watchdog/recursion-limit) remain unconditional.
- ScriptRunSummaryRecorder accumulates per-(instance, script) run counters and a
new site-only ScriptRunSummaryFlushService emits one aggregate "script" Info
site event per ScriptRunSummaryIntervalSeconds (default 300s), top-50-script
breakdown with an "others" rollup, zero-activity intervals emit nothing.
- Per-script opt-in via PerRunScriptEventScripts ("Instance/Script" exact or
"Instance/*" wildcard), matched by the new pure ScriptRunEventPolicy. All three
options are read from IOptionsMonitor<SiteRuntimeOptions> per run, so the
policy is hot-togglable without a restart.
- Fixed the stale "event log is not replicated" comment at AkkaHostedService.cs
(~905): site_events IS registered in SiteLocalDbSetup.ReplicatedTables — the
singleton is what makes queries always hit the actively-written copy;
replication is what gives the singleton history to read after a failover
(memo Decision (b)). site_events replication itself is unchanged (still
registered) and already pinned by
tests/ZB.MOM.WW.ScadaBridge.Host.Tests/SiteLocalDbCdcRegistrationTests.cs.
- Updated Component-SiteEventLogging.md (Volume Policy section, corrected
Storage/replication rationale) and Component-SiteRuntime.md (Script Run
Launch + Error Handling sections).
This commit is contained in:
@@ -902,10 +902,15 @@ akka {{
|
||||
// Register local handlers with SiteCommunicationActor
|
||||
siteCommActor.Tell(new RegisterLocalHandler(LocalHandlerType.Artifacts, dmProxy));
|
||||
|
||||
// Event log handler — cluster singleton so queries always reach the
|
||||
// active node. The event log is node-local SQLite and is not
|
||||
// replicated; only the active node records events. A per-node handler
|
||||
// would let a central query land on the standby and find nothing.
|
||||
// Event log handler — cluster singleton so queries always reach the copy that is
|
||||
// ACTIVELY BEING WRITTEN: the site_events table IS replicated (LocalDb Phase 1,
|
||||
// SiteLocalDbSetup.ReplicatedTables), but replication alone would not save a
|
||||
// per-node handler — central queries route through SiteCommandDispatcher to
|
||||
// whichever node answers, and a standby's replica sits idle until failover. The
|
||||
// singleton is what guarantees queries always read the actively-written copy;
|
||||
// replication is what gives the singleton HISTORY to read after it moves —
|
||||
// without it, a failover would restart the event log empty at exactly the moment
|
||||
// an operator is investigating an incident (WP3.2 design memo, Decision (b)).
|
||||
var eventLogQueryService = _serviceProvider.GetService<SiteEventLogging.IEventLogQueryService>();
|
||||
if (eventLogQueryService != null)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2 (site_events volume policy — docs/plans/2026-08-15-site-events-policy-design.md):
|
||||
/// decides whether an instance script run should emit the legacy per-run Started/Completed
|
||||
/// Info site events, or stay silent and let <see cref="ScriptRunSummaryRecorder"/>'s interval
|
||||
/// aggregate carry the signal instead. Pure and side-effect-free — no DI, no actor context —
|
||||
/// so the three opt-in shapes (global switch, exact per-script match, per-instance wildcard)
|
||||
/// are unit-testable in isolation and reusable from <see cref="ScriptRunLauncher"/> without
|
||||
/// dragging config plumbing into the run body.
|
||||
/// </summary>
|
||||
public static class ScriptRunEventPolicy
|
||||
{
|
||||
/// <summary>
|
||||
/// True when per-run Started/Completed Info events should be emitted for this run.
|
||||
/// </summary>
|
||||
/// <param name="options">
|
||||
/// The effective <see cref="SiteRuntimeOptions"/> snapshot for this run (the caller reads
|
||||
/// <c>IOptionsMonitor<SiteRuntimeOptions>.CurrentValue</c> per run so this stays
|
||||
/// hot-togglable — see <see cref="SiteRuntimeOptions.PerRunScriptEvents"/>).
|
||||
/// </param>
|
||||
/// <param name="instanceName">The owning instance's name.</param>
|
||||
/// <param name="scriptName">The script's name.</param>
|
||||
/// <returns>
|
||||
/// <c>true</c> if <see cref="SiteRuntimeOptions.PerRunScriptEvents"/> is set, or if
|
||||
/// <see cref="SiteRuntimeOptions.PerRunScriptEventScripts"/> contains an exact
|
||||
/// <c>"{instanceName}/{scriptName}"</c> entry or an <c>"{instanceName}/*"</c> wildcard
|
||||
/// entry for this instance; otherwise <c>false</c>.
|
||||
/// </returns>
|
||||
public static bool ShouldEmitPerRun(SiteRuntimeOptions options, string instanceName, string scriptName)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
if (options.PerRunScriptEvents)
|
||||
return true;
|
||||
|
||||
var scripts = options.PerRunScriptEventScripts;
|
||||
if (scripts is not { Count: > 0 })
|
||||
return false;
|
||||
|
||||
var exact = $"{instanceName}/{scriptName}";
|
||||
var wildcard = $"{instanceName}/*";
|
||||
|
||||
foreach (var entry in scripts)
|
||||
{
|
||||
if (string.Equals(entry, exact, StringComparison.Ordinal) ||
|
||||
string.Equals(entry, wildcard, StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
using System.Diagnostics;
|
||||
using Akka.Actor;
|
||||
using Microsoft.CodeAnalysis.Scripting;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
|
||||
@@ -221,6 +223,20 @@ internal static class ScriptRunLauncher
|
||||
// it is available to the catch blocks regardless of scope state.
|
||||
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
|
||||
|
||||
// WP3.2 (site_events volume policy): both are singletons, resolved from the root
|
||||
// provider up front so they too are available to the catch blocks. optionsMonitor is
|
||||
// null in tests that pass no serviceProvider (SingleServiceProvider et al.) — the
|
||||
// constructor-passed `options` is the fallback, exactly like siteEventLogger's null
|
||||
// path degrades to "no site event log". Reading CurrentValue HERE (once per run, at
|
||||
// the top) rather than re-reading it at each call site is what "hot-togglable" means
|
||||
// in practice: an operator's live edit to PerRunScriptEvents/PerRunScriptEventScripts
|
||||
// takes effect on the NEXT run, without a restart.
|
||||
var summaryRecorder = serviceProvider?.GetService<ScriptRunSummaryRecorder>();
|
||||
var optionsMonitor = serviceProvider?.GetService<IOptionsMonitor<SiteRuntimeOptions>>();
|
||||
var effectiveOptions = optionsMonitor?.CurrentValue ?? options;
|
||||
var emitPerRunEvents = ScriptRunEventPolicy.ShouldEmitPerRun(effectiveOptions, instanceName, scriptName);
|
||||
var runStopwatch = Stopwatch.StartNew();
|
||||
|
||||
// WP3.1: identify the worker this body is holding, so the watchdog can detach and
|
||||
// replace it if the body wedges. Read in the FIRST synchronous segment — a script
|
||||
// that later hops threads on an await no longer holds this worker, and the run-stamp
|
||||
@@ -350,12 +366,19 @@ internal static class ScriptRunLauncher
|
||||
Scope = scope
|
||||
};
|
||||
|
||||
// Operational `script` event — execution started. Fire-and-forget
|
||||
// (the `_ =` discards the task) so the event log can never block or
|
||||
// fault the script's own run; mirrors the existing Error-path emit.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' started");
|
||||
// WP3.2: the aggregate ALWAYS records this run (that's the point of the interval
|
||||
// summary), but the per-run Info event itself is gated by ScriptRunEventPolicy —
|
||||
// off by default, on globally via PerRunScriptEvents, or on for named scripts via
|
||||
// PerRunScriptEventScripts. Fire-and-forget (the `_ =` discards the task) so the
|
||||
// event log can never block or fault the script's own run; mirrors the existing
|
||||
// Error-path emit, which stays unconditional regardless of this policy.
|
||||
summaryRecorder?.RecordStarted(instanceName, scriptName);
|
||||
if (emitPerRunEvents)
|
||||
{
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' started");
|
||||
}
|
||||
|
||||
var state = await compiledScript.RunAsync(globals, cts.Token);
|
||||
|
||||
@@ -365,10 +388,15 @@ internal static class ScriptRunLauncher
|
||||
replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
|
||||
}
|
||||
|
||||
// Operational `script` event — execution completed successfully.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' completed");
|
||||
// Operational `script` event — execution completed successfully. Same gating as
|
||||
// the started event above.
|
||||
summaryRecorder?.RecordCompleted(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
if (emitPerRunEvents)
|
||||
{
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Info", instanceName, $"ScriptActor:{scriptName}",
|
||||
$"Script '{scriptName}' on instance '{instanceName}' completed");
|
||||
}
|
||||
|
||||
// Notify the owning ScriptActor of completion (also releases its in-flight slot).
|
||||
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, true, null));
|
||||
@@ -376,10 +404,12 @@ internal static class ScriptRunLauncher
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
summaryRecorder?.RecordTimedOut(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' timed out after {timeout.TotalSeconds}s";
|
||||
logger.LogWarning("{Message} (run {RunId})", errorMsg, runId);
|
||||
|
||||
// Failures recorded to site event log; script NOT disabled after failure.
|
||||
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
|
||||
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
|
||||
_ = siteEventLogger?.LogEventAsync(
|
||||
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
|
||||
|
||||
@@ -393,7 +423,9 @@ internal static class ScriptRunLauncher
|
||||
catch (Exception ex)
|
||||
{
|
||||
healthCollector?.IncrementScriptError();
|
||||
// Failures recorded to site event log; script NOT disabled after failure.
|
||||
summaryRecorder?.RecordFailed(instanceName, scriptName, runStopwatch.ElapsedMilliseconds);
|
||||
// Failures recorded to site event log; script NOT disabled after failure. WP3.2:
|
||||
// Error-level emissions are unconditional — never gated by ScriptRunEventPolicy.
|
||||
var errorMsg = $"Script '{scriptName}' on instance '{instanceName}' failed: {ex.Message}";
|
||||
logger.LogError(ex, "Script execution failed: {Script} on {Instance} (run {RunId})",
|
||||
scriptName, instanceName, runId);
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2 (site_events volume policy — docs/plans/2026-08-15-site-events-policy-design.md):
|
||||
/// site-only hosted service that periodically flushes <see cref="ScriptRunSummaryRecorder"/>
|
||||
/// into one aggregate "script" Info site event. Registered by
|
||||
/// <c>ServiceCollectionExtensions.AddSiteRuntime</c>, which only the site composition root
|
||||
/// calls (<c>SiteServiceRegistration.Configure</c>) — central never runs scripts, so central
|
||||
/// never registers this service, which is what makes it "Site nodes only" per the design memo.
|
||||
///
|
||||
/// <para>The interval is read from <see cref="IOptionsMonitor{TOptions}"/> on every tick (not
|
||||
/// captured once at construction) so <see cref="SiteRuntimeOptions.ScriptRunSummaryIntervalSeconds"/>
|
||||
/// is hot-reloadable — an operator can shorten or lengthen it, or disable it (<c>0</c>), live.
|
||||
/// Mirrors <c>ScriptSchedulerStatsReporter</c>'s shape: fixed cadence, exceptions logged and
|
||||
/// swallowed so the loop survives every flush failure.</para>
|
||||
/// </summary>
|
||||
public sealed class ScriptRunSummaryFlushService : BackgroundService
|
||||
{
|
||||
/// <summary>Poll cadence used while summaries are disabled (<c>ScriptRunSummaryIntervalSeconds == 0</c>) — coarse enough to amortise re-checking whether the interval was re-enabled live.</summary>
|
||||
private static readonly TimeSpan DisabledPollInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly ScriptRunSummaryRecorder _recorder;
|
||||
private readonly ISiteEventLogger _siteEventLogger;
|
||||
private readonly IOptionsMonitor<SiteRuntimeOptions> _optionsMonitor;
|
||||
private readonly ILogger<ScriptRunSummaryFlushService> _logger;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="ScriptRunSummaryFlushService"/>.</summary>
|
||||
/// <param name="recorder">The recorder whose accumulated counters this service flushes.</param>
|
||||
/// <param name="siteEventLogger">The site event logger the flushed summary row is written to.</param>
|
||||
/// <param name="optionsMonitor">Supplies the hot-reloadable flush interval.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public ScriptRunSummaryFlushService(
|
||||
ScriptRunSummaryRecorder recorder,
|
||||
ISiteEventLogger siteEventLogger,
|
||||
IOptionsMonitor<SiteRuntimeOptions> optionsMonitor,
|
||||
ILogger<ScriptRunSummaryFlushService> logger)
|
||||
{
|
||||
_recorder = recorder ?? throw new ArgumentNullException(nameof(recorder));
|
||||
_siteEventLogger = siteEventLogger ?? throw new ArgumentNullException(nameof(siteEventLogger));
|
||||
_optionsMonitor = optionsMonitor ?? throw new ArgumentNullException(nameof(optionsMonitor));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var intervalSeconds = _optionsMonitor.CurrentValue.ScriptRunSummaryIntervalSeconds;
|
||||
var wait = intervalSeconds > 0 ? TimeSpan.FromSeconds(intervalSeconds) : DisabledPollInterval;
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(wait, stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (intervalSeconds <= 0)
|
||||
{
|
||||
// Summaries disabled for this tick's window: counters keep accumulating in
|
||||
// the recorder (harmless — the next enabled flush just reports a longer
|
||||
// window), we simply don't emit or reset yet.
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _recorder.FlushAsync(_siteEventLogger).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ScriptRunSummaryFlushService flush failed; next tick will retry.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2 (site_events volume policy — docs/plans/2026-08-15-site-events-policy-design.md):
|
||||
/// accumulates per-(instance, script) run counters between flushes and emits ONE aggregate
|
||||
/// "script" Info site event per interval instead of the two per-run Started/Completed rows
|
||||
/// that were the dominant <c>site_events</c> writer. Registered as a process-wide singleton
|
||||
/// (<c>ServiceCollectionExtensions.AddSiteRuntime</c>); incremented from the same
|
||||
/// <see cref="ScriptRunLauncher"/> call sites that used to emit the per-run events, flushed
|
||||
/// on a timer by <c>ScriptRunSummaryFlushService</c>.
|
||||
///
|
||||
/// <para>Lock-free: each script's counters are a small class of <c>long</c> fields mutated
|
||||
/// via <see cref="Interlocked"/>, reached through a <see cref="ConcurrentDictionary{TKey,TValue}"/>
|
||||
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>) so a
|
||||
/// concurrent increment either lands in the snapshot being flushed or the fresh one that
|
||||
/// replaces it — never lost, never double-counted, and the flush never blocks a recording
|
||||
/// call (or vice versa).</para>
|
||||
/// </summary>
|
||||
public sealed class ScriptRunSummaryRecorder
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-script breakdown cap in the flushed details JSON — the rest fold into a single
|
||||
/// "others" rollup entry so a deployment with hundreds of scripts cannot mint an
|
||||
/// oversized site event row.
|
||||
/// </summary>
|
||||
internal const int TopScriptCap = 50;
|
||||
|
||||
private sealed class Counters
|
||||
{
|
||||
public long Started;
|
||||
public long Completed;
|
||||
public long Failed;
|
||||
public long TimedOut;
|
||||
public long TotalDurationMs;
|
||||
public long MaxDurationMs;
|
||||
}
|
||||
|
||||
private ConcurrentDictionary<(string InstanceName, string ScriptName), Counters> _counters = new();
|
||||
|
||||
private Counters GetOrAddCounters(string instanceName, string scriptName) =>
|
||||
Volatile.Read(ref _counters).GetOrAdd((instanceName, scriptName), static _ => new Counters());
|
||||
|
||||
/// <summary>Records that a script run began executing (past the shed-at-dequeue check).</summary>
|
||||
public void RecordStarted(string instanceName, string scriptName)
|
||||
{
|
||||
var counters = GetOrAddCounters(instanceName, scriptName);
|
||||
Interlocked.Increment(ref counters.Started);
|
||||
}
|
||||
|
||||
/// <summary>Records that a script run completed successfully.</summary>
|
||||
public void RecordCompleted(string instanceName, string scriptName, long durationMs)
|
||||
{
|
||||
var counters = GetOrAddCounters(instanceName, scriptName);
|
||||
Interlocked.Increment(ref counters.Completed);
|
||||
RecordDuration(counters, durationMs);
|
||||
}
|
||||
|
||||
/// <summary>Records that a script run threw an unhandled exception.</summary>
|
||||
public void RecordFailed(string instanceName, string scriptName, long durationMs)
|
||||
{
|
||||
var counters = GetOrAddCounters(instanceName, scriptName);
|
||||
Interlocked.Increment(ref counters.Failed);
|
||||
RecordDuration(counters, durationMs);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records that a script run timed out — including a run shed at dequeue (its deadline
|
||||
/// was already spent queueing, so it never reached <see cref="RecordStarted"/>).
|
||||
/// </summary>
|
||||
public void RecordTimedOut(string instanceName, string scriptName, long durationMs)
|
||||
{
|
||||
var counters = GetOrAddCounters(instanceName, scriptName);
|
||||
Interlocked.Increment(ref counters.TimedOut);
|
||||
RecordDuration(counters, durationMs);
|
||||
}
|
||||
|
||||
private static void RecordDuration(Counters counters, long durationMs)
|
||||
{
|
||||
if (durationMs < 0) durationMs = 0;
|
||||
Interlocked.Add(ref counters.TotalDurationMs, durationMs);
|
||||
InterlockedMax(ref counters.MaxDurationMs, durationMs);
|
||||
}
|
||||
|
||||
/// <summary>Lock-free running-max update (no built-in <c>Interlocked.Max(ref long, long)</c>).</summary>
|
||||
private static void InterlockedMax(ref long location, long candidate)
|
||||
{
|
||||
var initial = Volatile.Read(ref location);
|
||||
while (candidate > initial)
|
||||
{
|
||||
var previous = Interlocked.CompareExchange(ref location, candidate, initial);
|
||||
if (previous == initial) return;
|
||||
initial = previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Snapshots and resets the accumulated counters, then — if any script ran, failed, or
|
||||
/// timed out during the interval — emits ONE aggregate "script" Info site event
|
||||
/// summarizing the window. An interval with zero activity emits no row (the standby,
|
||||
/// which never runs scripts, therefore never produces a summary row naturally).
|
||||
/// </summary>
|
||||
/// <param name="siteEventLogger">The site event logger to emit the aggregate row to.</param>
|
||||
/// <returns><c>true</c> if a summary row was emitted; <c>false</c> if the interval was idle.</returns>
|
||||
public async Task<bool> FlushAsync(ISiteEventLogger siteEventLogger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(siteEventLogger);
|
||||
|
||||
// The replacement dictionary's type argument must name its tuple elements
|
||||
// IDENTICALLY to the field's declared type — Interlocked.Exchange<T> is generic
|
||||
// over both the ref and value parameters, and mixing a named-tuple exact bound
|
||||
// with an unnamed one erases the names from the inferred T (and therefore from
|
||||
// `snapshot` below).
|
||||
var snapshot = Interlocked.Exchange(
|
||||
ref _counters,
|
||||
new ConcurrentDictionary<(string InstanceName, string ScriptName), Counters>());
|
||||
|
||||
var entries = snapshot
|
||||
.Where(kvp => kvp.Value.Started > 0 || kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
|
||||
.ToList();
|
||||
|
||||
if (entries.Count == 0)
|
||||
return false;
|
||||
|
||||
long totalStarted = 0, totalCompleted = 0, totalFailed = 0, totalTimedOut = 0;
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
totalStarted += entry.Value.Started;
|
||||
totalCompleted += entry.Value.Completed;
|
||||
totalFailed += entry.Value.Failed;
|
||||
totalTimedOut += entry.Value.TimedOut;
|
||||
}
|
||||
|
||||
// Rank by total run activity. Started+TimedOut (not just Started): a run shed at
|
||||
// dequeue never reaches RecordStarted, so ranking on Started alone would silently
|
||||
// drop a script that does nothing BUT get shed from the breakdown entirely.
|
||||
var ranked = entries
|
||||
.OrderByDescending(e => e.Value.Started + e.Value.TimedOut)
|
||||
.ToList();
|
||||
|
||||
var top = ranked.Take(TopScriptCap);
|
||||
var rest = ranked.Skip(TopScriptCap).ToList();
|
||||
|
||||
var breakdown = top.Select(e => new ScriptSummaryEntry(
|
||||
e.Key.InstanceName,
|
||||
e.Key.ScriptName,
|
||||
e.Value.Started,
|
||||
e.Value.Completed,
|
||||
e.Value.Failed,
|
||||
e.Value.TimedOut,
|
||||
e.Value.Completed > 0 ? e.Value.TotalDurationMs / e.Value.Completed : 0,
|
||||
e.Value.MaxDurationMs)).ToList();
|
||||
|
||||
object details;
|
||||
if (rest.Count > 0)
|
||||
{
|
||||
details = new
|
||||
{
|
||||
scripts = breakdown,
|
||||
others = new
|
||||
{
|
||||
scriptCount = rest.Count,
|
||||
started = rest.Sum(e => e.Value.Started),
|
||||
completed = rest.Sum(e => e.Value.Completed),
|
||||
failed = rest.Sum(e => e.Value.Failed),
|
||||
timedOut = rest.Sum(e => e.Value.TimedOut),
|
||||
}
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
details = new { scripts = breakdown };
|
||||
}
|
||||
|
||||
var message = string.Format(
|
||||
CultureInfo.InvariantCulture,
|
||||
"{0:N0} runs: {1:N0} completed, {2:N0} failed, {3:N0} timed out across {4:N0} scripts",
|
||||
totalStarted, totalCompleted, totalFailed, totalTimedOut, entries.Count);
|
||||
|
||||
var detailsJson = JsonSerializer.Serialize(details, DetailsJsonOptions);
|
||||
|
||||
await siteEventLogger.LogEventAsync(
|
||||
"script", "Info", null, "ScriptRunSummary", message, detailsJson).ConfigureAwait(false);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Built once per assembly (WP1.5 convention) rather than per flush.
|
||||
private static readonly JsonSerializerOptions DetailsJsonOptions =
|
||||
new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
|
||||
|
||||
private sealed record ScriptSummaryEntry(
|
||||
string InstanceName,
|
||||
string ScriptName,
|
||||
long Started,
|
||||
long Completed,
|
||||
long Failed,
|
||||
long TimedOut,
|
||||
long AvgDurationMs,
|
||||
long MaxDurationMs);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.LocalDb;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Persistence;
|
||||
using ZB.MOM.WW.ScadaBridge.SiteRuntime.Repositories;
|
||||
@@ -98,6 +99,18 @@ public static class ServiceCollectionExtensions
|
||||
sp.GetRequiredService<SiteStreamManager>(),
|
||||
sp.GetRequiredService<ILogger<Streaming.SiteStreamAlarmDropReporter>>()));
|
||||
|
||||
// WP3.2 (site_events volume policy): the recorder is a singleton so ScriptRunLauncher
|
||||
// can resolve it from the root service provider on every run, the same way it already
|
||||
// resolves ISiteEventLogger. The flush service is registered ONLY here — central
|
||||
// composition roots never call AddSiteRuntime(), which is what makes the periodic
|
||||
// "script" summary event site-only, matching the design memo.
|
||||
services.AddSingleton<ScriptRunSummaryRecorder>();
|
||||
services.AddHostedService(sp => new ScriptRunSummaryFlushService(
|
||||
sp.GetRequiredService<ScriptRunSummaryRecorder>(),
|
||||
sp.GetRequiredService<ISiteEventLogger>(),
|
||||
sp.GetRequiredService<IOptionsMonitor<SiteRuntimeOptions>>(),
|
||||
sp.GetRequiredService<ILogger<ScriptRunSummaryFlushService>>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
|
||||
@@ -141,4 +141,44 @@ public class SiteRuntimeOptions
|
||||
/// <see cref="StreamBufferSize"/>).
|
||||
/// </summary>
|
||||
public int AlarmPublishQueueCapacity { get; set; } = 2000;
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2 (site_events volume policy — docs/plans/2026-08-15-site-events-policy-design.md):
|
||||
/// global switch restoring the legacy per-run Started/Completed Info site events for
|
||||
/// EVERY instance script run. Off by default: those two rows per run were the dominant
|
||||
/// <c>site_events</c> writer (~12 physical row-writes across a replicated pair per run,
|
||||
/// counting CDC oplog + row-version rows and their eventual purge tombstones), for
|
||||
/// content nobody looks at unless something went wrong. With this off, an interval
|
||||
/// aggregate (<see cref="Scripts.ScriptRunSummaryRecorder"/>) replaces the per-run rows
|
||||
/// and script Error events (timeout/failure/stuck-watchdog) are unaffected — they are
|
||||
/// unconditional regardless of this switch. Read via
|
||||
/// <see cref="Microsoft.Extensions.Options.IOptionsMonitor{TOptions}"/> at the top of
|
||||
/// every run, so it is hot-togglable (no restart, no redeploy). Default: false.
|
||||
/// </summary>
|
||||
public bool PerRunScriptEvents { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2: per-script opt-in restoring the legacy per-run Started/Completed Info site
|
||||
/// events for NAMED scripts only, for live debugging without flipping the global
|
||||
/// <see cref="PerRunScriptEvents"/> switch (and therefore without paying its volume
|
||||
/// cost for every other script). Effective only while the global switch is off.
|
||||
/// Entries are <c>"{InstanceName}/{ScriptName}"</c> for an exact match, or
|
||||
/// <c>"{InstanceName}/*"</c> to opt in every script on that instance. Matched by
|
||||
/// <see cref="Scripts.ScriptRunEventPolicy.ShouldEmitPerRun"/>, read from
|
||||
/// <see cref="Microsoft.Extensions.Options.IOptionsMonitor{TOptions}"/> per run — hot
|
||||
/// -togglable like <see cref="PerRunScriptEvents"/>. Default: empty (no opt-ins).
|
||||
/// </summary>
|
||||
public List<string> PerRunScriptEventScripts { get; set; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// WP3.2: flush interval (seconds) for <see cref="Scripts.ScriptRunSummaryRecorder"/>'s
|
||||
/// interval aggregate — one "script" Info site event per interval summarizing every
|
||||
/// instance script run since the last flush (headline totals + a top-50-scripts
|
||||
/// breakdown, capped so a pathological deployment cannot mint an oversized row). An
|
||||
/// interval with zero activity emits no row. <c>0</c> disables the summary flush
|
||||
/// entirely (the recorder still accumulates counters; they are just never emitted or
|
||||
/// reset). Default: 300 (5 minutes) — at most 288 summary rows/day, versus 2 rows per
|
||||
/// script run under the legacy per-run behaviour.
|
||||
/// </summary>
|
||||
public int ScriptRunSummaryIntervalSeconds { get; set; } = 300;
|
||||
}
|
||||
|
||||
@@ -86,5 +86,10 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
|
||||
builder.RequireThat(options.AlarmPublishQueueCapacity > 0,
|
||||
$"ScadaBridge:SiteRuntime:AlarmPublishQueueCapacity must be greater than 0 " +
|
||||
$"(was {options.AlarmPublishQueueCapacity}); it bounds SiteStreamManager's dedicated alarm publish queue.");
|
||||
|
||||
builder.RequireThat(options.ScriptRunSummaryIntervalSeconds >= 0,
|
||||
$"ScadaBridge:SiteRuntime:ScriptRunSummaryIntervalSeconds must be >= 0 " +
|
||||
$"(was {options.ScriptRunSummaryIntervalSeconds}); it is the ScriptRunSummaryRecorder " +
|
||||
"flush interval and a negative value throws inside its delay. 0 disables the flush.");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user