ScadaBridge/src/ZB.MOM.WW.ScadaBridge.SiteRuntime/Scripts/ScriptRunSummaryRecorder.cs

using System.Collections.Concurrent;
using System.Globalization;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;

namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;

/// <summary>
/// Site_events volume policy (docs/plans/2026-08-15-site-events-policy-design.md):
/// accumulates per-(instance, script) run counters between flushes and emits ONE aggregate
/// "script" Info site event per interval instead of the two per-run Started/Completed rows
/// that were the dominant <c>site_events</c> writer. Registered as a process-wide singleton
/// (<c>ServiceCollectionExtensions.AddSiteRuntime</c>); incremented from the same
/// <see cref="ScriptRunLauncher"/> call sites that used to emit the per-run events, flushed
/// on a timer by <c>ScriptRunSummaryFlushService</c>.
///
/// <para>Lock-free: each script's counters are a small class of <c>long</c> fields mutated
/// via <see cref="Interlocked"/>, reached through a <see cref="ConcurrentDictionary{TKey,TValue}"/>
/// that <see cref="FlushAsync"/> atomically swaps out (<see cref="Interlocked.Exchange"/>), so
/// the flush never blocks a recording call (or vice versa).</para>
///
/// <para>Accounting is BEST-EFFORT, not exact. A recorder that has already resolved its
/// <see cref="Counters"/> object from the pre-swap dictionary can increment it after the swap
/// has snapshotted it, so that increment lands in a window that has already been summarised
/// and is never reported. These are operational volume counters for one interval row, not a
/// ledger: an occasional count landing in neither window is acceptable, and nothing
/// downstream reconciles totals across intervals. What IS guaranteed is that a count is never
/// double-reported (each <see cref="Counters"/> instance is summarised by exactly one flush)
/// and that runs spanning a flush boundary — started in window N, completed in N+1 — still
/// emit their completion, because the flush filter admits a completion-only entry.</para>
/// </summary>
public sealed class ScriptRunSummaryRecorder
{
    /// <summary>
    /// Per-script breakdown cap in the flushed details JSON — the rest fold into a single
    /// "others" rollup entry so a deployment with hundreds of scripts cannot mint an
    /// oversized site event row.
    /// </summary>
    internal const int TopScriptCap = 50;

    private sealed class Counters
    {
        public long Started;
        public long Completed;
        public long Failed;
        public long TimedOut;
        public long TotalDurationMs;
        public long MaxDurationMs;
    }

    private ConcurrentDictionary<(string InstanceName, string ScriptName), Counters> _counters = new();

    private Counters GetOrAddCounters(string instanceName, string scriptName) =>
        Volatile.Read(ref _counters).GetOrAdd((instanceName, scriptName), static _ => new Counters());

    /// <summary>Records that a script run began executing (past the shed-at-dequeue check).</summary>
    /// <param name="instanceName">The instance the script belongs to.</param>
    /// <param name="scriptName">The script that ran.</param>
    public void RecordStarted(string instanceName, string scriptName)
    {
        var counters = GetOrAddCounters(instanceName, scriptName);
        Interlocked.Increment(ref counters.Started);
    }

    /// <summary>Records that a script run completed successfully.</summary>
    /// <param name="instanceName">The instance the script belongs to.</param>
    /// <param name="scriptName">The script that ran.</param>
    /// <param name="durationMs">The run duration in milliseconds.</param>
    public void RecordCompleted(string instanceName, string scriptName, long durationMs)
    {
        var counters = GetOrAddCounters(instanceName, scriptName);
        Interlocked.Increment(ref counters.Completed);
        RecordDuration(counters, durationMs);
    }

    /// <summary>Records that a script run threw an unhandled exception.</summary>
    /// <param name="instanceName">The instance the script belongs to.</param>
    /// <param name="scriptName">The script that ran.</param>
    /// <param name="durationMs">The run duration in milliseconds.</param>
    public void RecordFailed(string instanceName, string scriptName, long durationMs)
    {
        var counters = GetOrAddCounters(instanceName, scriptName);
        Interlocked.Increment(ref counters.Failed);
        RecordDuration(counters, durationMs);
    }

    /// <summary>
    /// Records that a script run timed out — including a run shed at dequeue (its deadline
    /// was already spent queueing, so it never reached <see cref="RecordStarted"/>).
    /// </summary>
    /// <param name="instanceName">The instance the script belongs to.</param>
    /// <param name="scriptName">The script that timed out.</param>
    /// <param name="durationMs">The run duration in milliseconds.</param>
    public void RecordTimedOut(string instanceName, string scriptName, long durationMs)
    {
        var counters = GetOrAddCounters(instanceName, scriptName);
        Interlocked.Increment(ref counters.TimedOut);
        RecordDuration(counters, durationMs);
    }

    private static void RecordDuration(Counters counters, long durationMs)
    {
        if (durationMs < 0) durationMs = 0;
        Interlocked.Add(ref counters.TotalDurationMs, durationMs);
        InterlockedMax(ref counters.MaxDurationMs, durationMs);
    }

    /// <summary>Lock-free running-max update (no built-in <c>Interlocked.Max(ref long, long)</c>).</summary>
    private static void InterlockedMax(ref long location, long candidate)
    {
        var initial = Volatile.Read(ref location);
        while (candidate > initial)
        {
            var previous = Interlocked.CompareExchange(ref location, candidate, initial);
            if (previous == initial) return;
            initial = previous;
        }
    }

    /// <summary>
    /// Snapshots and resets the accumulated counters, then — if any script started, completed,
    /// failed, or timed out during the interval — emits ONE aggregate "script" Info site event
    /// summarizing the window. An interval with zero activity emits no row (the standby,
    /// which never runs scripts, therefore never produces a summary row naturally).
    /// </summary>
    /// <param name="siteEventLogger">The site event logger to emit the aggregate row to.</param>
    /// <returns><c>true</c> if a summary row was emitted; <c>false</c> if the interval was idle.</returns>
    public async Task<bool> FlushAsync(ISiteEventLogger siteEventLogger)
    {
        ArgumentNullException.ThrowIfNull(siteEventLogger);

        // The replacement dictionary's type argument must name its tuple elements
        // IDENTICALLY to the field's declared type — Interlocked.Exchange<T> is generic
        // over both the ref and value parameters, and mixing a named-tuple exact bound
        // with an unnamed one erases the names from the inferred T (and therefore from
        // `snapshot` below).
        var snapshot = Interlocked.Exchange(
            ref _counters,
            new ConcurrentDictionary<(string InstanceName, string ScriptName), Counters>());

        // Completed > 0 is admitted on its own: a run that started in window N and finished in
        // N+1 records its completion against a FRESH counters object whose Started is 0, so
        // filtering on Started/Failed/TimedOut alone would silently drop that completion (and,
        // with it, the run's duration) from every summary row that could ever carry it.
        var entries = snapshot
            .Where(kvp => kvp.Value.Started > 0 || kvp.Value.Completed > 0 ||
                          kvp.Value.Failed > 0 || kvp.Value.TimedOut > 0)
            .ToList();

        if (entries.Count == 0)
            return false;

        long totalStarted = 0, totalCompleted = 0, totalFailed = 0, totalTimedOut = 0;
        foreach (var entry in entries)
        {
            totalStarted += entry.Value.Started;
            totalCompleted += entry.Value.Completed;
            totalFailed += entry.Value.Failed;
            totalTimedOut += entry.Value.TimedOut;
        }

        // Rank by total run activity. Started+TimedOut (not just Started): a run shed at
        // dequeue never reaches RecordStarted, so ranking on Started alone would silently
        // drop a script that does nothing BUT get shed from the breakdown entirely.
        var ranked = entries
            .OrderByDescending(e => e.Value.Started + e.Value.TimedOut)
            .ToList();

        var top = ranked.Take(TopScriptCap);
        var rest = ranked.Skip(TopScriptCap).ToList();

        var breakdown = top.Select(e => new ScriptSummaryEntry(
            e.Key.InstanceName,
            e.Key.ScriptName,
            e.Value.Started,
            e.Value.Completed,
            e.Value.Failed,
            e.Value.TimedOut,
            e.Value.Completed > 0 ? e.Value.TotalDurationMs / e.Value.Completed : 0,
            e.Value.MaxDurationMs)).ToList();

        object details;
        if (rest.Count > 0)
        {
            details = new
            {
                scripts = breakdown,
                others = new
                {
                    scriptCount = rest.Count,
                    started = rest.Sum(e => e.Value.Started),
                    completed = rest.Sum(e => e.Value.Completed),
                    failed = rest.Sum(e => e.Value.Failed),
                    timedOut = rest.Sum(e => e.Value.TimedOut),
                }
            };
        }
        else
        {
            details = new { scripts = breakdown };
        }

        var message = string.Format(
            CultureInfo.InvariantCulture,
            "{0:N0} runs: {1:N0} completed, {2:N0} failed, {3:N0} timed out across {4:N0} scripts",
            totalStarted, totalCompleted, totalFailed, totalTimedOut, entries.Count);

        var detailsJson = JsonSerializer.Serialize(details, DetailsJsonOptions);

        await siteEventLogger.LogEventAsync(
            "script", "Info", null, "ScriptRunSummary", message, detailsJson).ConfigureAwait(false);

        return true;
    }

    // Built once per assembly rather than per flush.
    private static readonly JsonSerializerOptions DetailsJsonOptions =
        new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };

    private sealed record ScriptSummaryEntry(
        string InstanceName,
        string ScriptName,
        long Started,
        long Completed,
        long Failed,
        long TimedOut,
        long AvgDurationMs,
        long MaxDurationMs);
}