feat(site-runtime): traced runs emit ScriptRun rows, return ExecutionId + trace events; traced value-change runs inherit the parent

Flow Debugger (design 2026-09-05 §5.1/§5.4): ScriptRunLauncher emits one ScriptRun row per
traced run on every exit path (OccurredAtUtc = run start), replies with the run's
ExecutionId (always) and the collector snapshot (traced only). ScriptActor forwards
ScriptCallRequest.Trace and, for a value-change trigger fired by a traced stamped change,
parents the run to the writer with trigger detail in Extra; every other trigger stays a root.

Claude-Session: https://claude.ai/code/session_01GSLkkHKz9JrAtmJjmQK3cP
This commit is contained in:
Joseph Doherty
2026-09-05 23:43:33 -04:00
parent 93489cd7a2
commit eca1434160
3 changed files with 319 additions and 17 deletions
@@ -7,7 +7,10 @@ using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using ZB.MOM.WW.ScadaBridge.HealthMonitoring;
@@ -74,6 +77,8 @@ internal static class ScriptRunLauncher
/// <param name="parentExecutionId">ExecutionId of the spawning execution for audit correlation; null for root runs.</param>
/// <param name="executionTimeoutSeconds">Per-script execution timeout in seconds. Null or non-positive falls back to the global value.</param>
/// <param name="scheduler">Script-execution scheduler seam (#18); null selects the process-wide shared instance.</param>
/// <param name="trace">Flow Debugger trace flag (design 2026-09-05 §5.2): when true the run emits <c>ScriptTrace</c> rows and returns them inline on the <see cref="ScriptCallResult"/>.</param>
/// <param name="triggerExtra">Flow Debugger: trigger detail for a traced value-change run (stamped into the <c>ScriptRun</c> row's <c>Extra</c>); null for calls.</param>
public static void LaunchScript(
string scriptName,
string instanceName,
@@ -94,7 +99,9 @@ internal static class ScriptRunLauncher
IServiceProvider? serviceProvider,
Guid? parentExecutionId,
int? executionTimeoutSeconds,
ScriptExecutionScheduler? scheduler)
ScriptExecutionScheduler? scheduler,
bool trace = false,
ScriptTraceExtra? triggerExtra = null)
{
var timeout = ResolveTimeout(executionTimeoutSeconds, options);
var executionScheduler = scheduler ?? ScriptExecutionScheduler.Shared(options);
@@ -111,7 +118,7 @@ internal static class ScriptRunLauncher
instanceActor, sharedScriptLibrary, options, replyTo, correlationId,
completionTarget, siteCommunicationActor, logger, scope, runId,
healthCollector, serviceProvider, parentExecutionId, timeout,
executionScheduler, cts),
executionScheduler, cts, trace, triggerExtra),
CancellationToken.None, TaskCreationOptions.DenyChildAttach, executionScheduler)
.Unwrap();
}
@@ -231,9 +238,19 @@ internal static class ScriptRunLauncher
Guid? parentExecutionId,
TimeSpan timeout,
ScriptExecutionScheduler executionScheduler,
CancellationTokenSource cts)
CancellationTokenSource cts,
bool trace = false,
ScriptTraceExtra? triggerExtra = null)
{
IServiceScope? serviceScope = null;
// Flow Debugger (design §5.4): the inline sink exists only for a traced run; the
// ScriptRun row itself is emitted below on every exit path of a traced run.
var collector = trace ? new FlowTraceCollector() : null;
var startedUtc = DateTime.UtcNow;
ScriptRuntimeContext? context = null;
IAuditWriter? auditWriter = null;
var siteId = string.Empty;
string? sourceNode = null;
// ISiteEventLogger is a singleton; resolve from the root provider so
// it is available to the catch blocks regardless of scope state.
var siteEventLogger = serviceProvider?.GetService<ISiteEventLogger>();
@@ -281,11 +298,11 @@ internal static class ScriptRunLauncher
// Notification Outbox: the S&F engine is a singleton; the site identity
// provider supplies the site id stamped on enqueued notifications.
StoreAndForwardService? storeAndForward = null;
var siteId = string.Empty;
// The writer is a singleton (FallbackAuditWriter
// composes the SQLite hot-path + drop-oldest ring); null in tests / hosts
// that haven't called AddAuditLog, which the helper handles as a no-op.
IAuditWriter? auditWriter = null;
// (siteId / auditWriter / sourceNode are declared above the try so the
// traced ScriptRun emission in the catch blocks can reach them.)
// Site-local tracking store
// backing Tracking.Status(id). Singleton; null in tests / hosts
// that haven't wired the store, which the helper handles by
@@ -304,7 +321,6 @@ internal static class ScriptRunLauncher
// provider, in which case NotificationSubmit.SourceNode and
// SiteCallOperational.SourceNode stay null and central
// persists the rows with SourceNode NULL.
string? sourceNode = null;
if (serviceProvider != null)
{
@@ -320,7 +336,7 @@ internal static class ScriptRunLauncher
sourceNode = serviceScope.ServiceProvider.GetService<INodeIdentityProvider>()?.NodeName;
}
var context = new ScriptRuntimeContext(
context = new ScriptRuntimeContext(
instanceActor,
sharedScriptLibrary,
callDepth,
@@ -371,7 +387,10 @@ internal static class ScriptRunLauncher
// execution-timeout token so Attributes.WaitAsync's Ask is
// bounded by the script's own ExecutionTimeoutSeconds — a
// shorter script deadline wins over the wait's own timeout.
scriptTimeoutToken: cts.Token);
scriptTimeoutToken: cts.Token,
// Flow Debugger (design §5.2/§5.4): the trace flag and the inline sink.
isTraced: trace,
traceCollector: collector);
var globals = new ScriptGlobals
{
@@ -397,10 +416,18 @@ internal static class ScriptRunLauncher
var state = await compiledScript.RunAsync(globals, cts.Token);
// Flow Debugger: the ScriptRun row goes into the collector BEFORE the reply so the
// inline list the caller receives includes the run itself.
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
context.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
parameters, state.ReturnValue, error: null, triggerExtra);
// Send result to requester if this was an Ask-based call
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, true, state.ReturnValue, null));
replyTo.Tell(new ScriptCallResult(
correlationId, true, state.ReturnValue, null,
context.ExecutionId, collector?.Snapshot()));
}
// Operational `script` event — execution completed successfully. Same gating as
@@ -428,9 +455,14 @@ internal static class ScriptRunLauncher
_ = siteEventLogger?.LogEventAsync(
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg);
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
parameters, null, errorMsg, triggerExtra);
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
replyTo.Tell(new ScriptCallResult(
correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Snapshot()));
}
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
@@ -448,9 +480,14 @@ internal static class ScriptRunLauncher
_ = siteEventLogger?.LogEventAsync(
"script", "Error", instanceName, $"ScriptActor:{scriptName}", errorMsg, ex.ToString());
EmitScriptRun(trace, collector, auditWriter, logger, scriptName, instanceName, siteId, sourceNode,
context?.ExecutionId, parentExecutionId, startedUtc, runStopwatch.ElapsedMilliseconds,
parameters, null, errorMsg, triggerExtra);
if (!replyTo.IsNobody())
{
replyTo.Tell(new ScriptCallResult(correlationId, false, null, errorMsg));
replyTo.Tell(new ScriptCallResult(
correlationId, false, null, errorMsg, context?.ExecutionId, collector?.Snapshot()));
}
completionTarget.Tell(new ScriptActor.ScriptExecutionCompleted(scriptName, false, errorMsg));
@@ -466,6 +503,59 @@ internal static class ScriptRunLauncher
}
}
/// <summary>
/// Flow Debugger (design 2026-09-05 §5.1): emits the <c>ScriptTrace.ScriptRun</c> row for a
/// traced run — collector first (so the inline reply carries it), then the best-effort
/// audit writer. <c>OccurredAtUtc</c> is the run's START so the row sorts before the
/// writes/waits the run made. No-op when untraced; never throws.
/// </summary>
private static void EmitScriptRun(
bool trace, FlowTraceCollector? collector, IAuditWriter? auditWriter, ILogger logger,
string scriptName, string instanceName, string siteId, string? sourceNode,
Guid? executionId, Guid? parentExecutionId, DateTime startedUtc, long elapsedMs,
IReadOnlyDictionary<string, object?>? parameters, object? returnValue, string? error,
ScriptTraceExtra? triggerExtra)
{
if (!trace) return;
try
{
var sourceScript = $"ScriptActor:{scriptName}";
var evt = ScadaBridgeAuditEventFactory.Create(
AuditChannel.ScriptTrace, AuditKind.ScriptRun,
error is null ? AuditStatus.Delivered : AuditStatus.Failed,
occurredAtUtc: DateTime.SpecifyKind(startedUtc, DateTimeKind.Utc),
actor: sourceScript, target: scriptName, sourceNode: sourceNode,
executionId: executionId, parentExecutionId: parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(siteId) ? null : siteId,
sourceInstanceId: instanceName, sourceScript: sourceScript,
durationMs: (int)Math.Min(elapsedMs, int.MaxValue),
errorMessage: error,
requestSummary: TraceJson(parameters),
responseSummary: TraceJson(returnValue),
extra: (triggerExtra ?? new ScriptTraceExtra { TriggerKind = "call" }).ToJson());
collector?.Add(evt);
if (auditWriter is null) return;
var t = auditWriter.WriteAsync(evt, CancellationToken.None);
if (!t.IsCompletedSuccessfully)
{
_ = t.ContinueWith(
x => logger.LogWarning(x.Exception, "Flow trace ScriptRun write failed for {Script}", scriptName),
TaskContinuationOptions.OnlyOnFaulted);
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Flow trace ScriptRun emission failed for {Script}", scriptName);
}
}
private static string? TraceJson(object? value)
{
if (value is null) return null;
try { return JsonSerializer.Serialize(value, ScriptTraceExtra.JsonOptions); }
catch (Exception) { return null; }
}
private static async Task RunAlarmScriptAsync(
string alarmName,
string instanceName,