using System.Collections.Concurrent; using System.Globalization; using System.Text.Json; using ZB.MOM.WW.ScadaBridge.SiteEventLogging; namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts; /// /// Site_events volume policy (docs/plans/2026-08-15-site-events-policy-design.md): /// accumulates per-(instance, script) run counters between flushes and emits ONE aggregate /// "script" Info site event per interval instead of the two per-run Started/Completed rows /// that were the dominant site_events writer. Registered as a process-wide singleton /// (ServiceCollectionExtensions.AddSiteRuntime); incremented from the same /// call sites that used to emit the per-run events, flushed /// on a timer by ScriptRunSummaryFlushService. /// /// Lock-free: each script's counters are a small class of long fields mutated /// via , reached through a /// that atomically swaps out (), so /// the flush never blocks a recording call (or vice versa). /// /// Accounting is BEST-EFFORT, not exact. A recorder that has already resolved its /// object from the pre-swap dictionary can increment it after the swap /// has snapshotted it, so that increment lands in a window that has already been summarised /// and is never reported. These are operational volume counters for one interval row, not a /// ledger: an occasional count landing in neither window is acceptable, and nothing /// downstream reconciles totals across intervals. What IS guaranteed is that a count is never /// double-reported (each instance is summarised by exactly one flush) /// and that runs spanning a flush boundary — started in window N, completed in N+1 — still /// emit their completion, because the flush filter admits a completion-only entry. /// public sealed class ScriptRunSummaryRecorder { /// /// Per-script breakdown cap in the flushed details JSON — the rest fold into a single /// "others" rollup entry so a deployment with hundreds of scripts cannot mint an /// oversized site event row. /// internal const int TopScriptCap = 50; private sealed class Counters { public long Started; public long Completed; public long Failed; public long TimedOut; public long TotalDurationMs; public long MaxDurationMs; } private ConcurrentDictionary<(string InstanceName, string ScriptName), Counters> _counters = new(); private Counters GetOrAddCounters(string instanceName, string scriptName) => Volatile.Read(ref _counters).GetOrAdd((instanceName, scriptName), static _ => new Counters()); /// Records that a script run began executing (past the shed-at-dequeue check). /// The instance the script belongs to. /// The script that ran. public void RecordStarted(string instanceName, string scriptName) { var counters = GetOrAddCounters(instanceName, scriptName); Interlocked.Increment(ref counters.Started); } /// Records that a script run completed successfully. /// The instance the script belongs to. /// The script that ran. /// The run duration in milliseconds. public void RecordCompleted(string instanceName, string scriptName, long durationMs) { var counters = GetOrAddCounters(instanceName, scriptName); Interlocked.Increment(ref counters.Completed); RecordDuration(counters, durationMs); } /// Records that a script run threw an unhandled exception. /// The instance the script belongs to. /// The script that ran. /// The run duration in milliseconds. public void RecordFailed(string instanceName, string scriptName, long durationMs) { var counters = GetOrAddCounters(instanceName, scriptName); Interlocked.Increment(ref counters.Failed); RecordDuration(counters, durationMs); } /// /// Records that a script run timed out — including a run shed at dequeue (its deadline /// was already spent queueing, so it never reached ). /// /// The instance the script belongs to. /// The script that timed out. /// The run duration in milliseconds. public void RecordTimedOut(string instanceName, string scriptName, long durationMs) { var counters = GetOrAddCounters(instanceName, scriptName); Interlocked.Increment(ref counters.TimedOut); RecordDuration(counters, durationMs); } private static void RecordDuration(Counters counters, long durationMs) { if (durationMs < 0) durationMs = 0; Interlocked.Add(ref counters.TotalDurationMs, durationMs); InterlockedMax(ref counters.MaxDurationMs, durationMs); } /// Lock-free running-max update (no built-in Interlocked.Max(ref long, long)). private static void InterlockedMax(ref long location, long candidate) { var initial = Volatile.Read(ref location); while (candidate > initial) { var previous = Interlocked.CompareExchange(ref location, candidate, initial); if (previous == initial) return; initial = previous; } } /// /// Snapshots and resets the accumulated counters, then — if any script started, completed, /// failed, or timed out during the interval — emits ONE aggregate "script" Info site event /// summarizing the window. An interval with zero activity emits no row (the standby, /// which never runs scripts, therefore never produces a summary row naturally). /// /// The site event logger to emit the aggregate row to. /// true if a summary row was emitted; false if the interval was idle. public async Task FlushAsync(ISiteEventLogger siteEventLogger) { ArgumentNullException.ThrowIfNull(siteEventLogger); // The replacement dictionary's type argument must name its tuple elements // IDENTICALLY to the field's declared type — Interlocked.Exchange is generic // over both the ref and value parameters, and mixing a named-tuple exact bound // with an unnamed one erases the names from the inferred T (and therefore from // `snapshot` below). var snapshot = Interlocked.Exchange( ref _counters, new ConcurrentDictionary<(string InstanceName, string ScriptName), Counters>()); // Completed > 0 is admitted on its own: a run that started in window N and finished in // N+1 records its completion against a FRESH counters object whose Started is 0, so // filtering on Started/Failed/TimedOut alone would silently drop that completion (and, // with it, the run's duration) from every summary row that could ever carry it. var entries = snapshot .Where(kvp => kvp.Value.Started > 0 || kvp.Value.Completed > 0 || kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0) .ToList(); if (entries.Count == 0) return false; long totalStarted = 0, totalCompleted = 0, totalFailed = 0, totalTimedOut = 0; foreach (var entry in entries) { totalStarted += entry.Value.Started; totalCompleted += entry.Value.Completed; totalFailed += entry.Value.Failed; totalTimedOut += entry.Value.TimedOut; } // Rank by total run activity. Started+TimedOut (not just Started): a run shed at // dequeue never reaches RecordStarted, so ranking on Started alone would silently // drop a script that does nothing BUT get shed from the breakdown entirely. var ranked = entries .OrderByDescending(e => e.Value.Started + e.Value.TimedOut) .ToList(); var top = ranked.Take(TopScriptCap); var rest = ranked.Skip(TopScriptCap).ToList(); var breakdown = top.Select(e => new ScriptSummaryEntry( e.Key.InstanceName, e.Key.ScriptName, e.Value.Started, e.Value.Completed, e.Value.Failed, e.Value.TimedOut, e.Value.Completed > 0 ? e.Value.TotalDurationMs / e.Value.Completed : 0, e.Value.MaxDurationMs)).ToList(); object details; if (rest.Count > 0) { details = new { scripts = breakdown, others = new { scriptCount = rest.Count, started = rest.Sum(e => e.Value.Started), completed = rest.Sum(e => e.Value.Completed), failed = rest.Sum(e => e.Value.Failed), timedOut = rest.Sum(e => e.Value.TimedOut), } }; } else { details = new { scripts = breakdown }; } var message = string.Format( CultureInfo.InvariantCulture, "{0:N0} runs: {1:N0} completed, {2:N0} failed, {3:N0} timed out across {4:N0} scripts", totalStarted, totalCompleted, totalFailed, totalTimedOut, entries.Count); var detailsJson = JsonSerializer.Serialize(details, DetailsJsonOptions); await siteEventLogger.LogEventAsync( "script", "Info", null, "ScriptRunSummary", message, detailsJson).ConfigureAwait(false); return true; } // Built once per assembly rather than per flush. private static readonly JsonSerializerOptions DetailsJsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; private sealed record ScriptSummaryEntry( string InstanceName, string ScriptName, long Started, long Completed, long Failed, long TimedOut, long AvgDurationMs, long MaxDurationMs); }