using System.Diagnostics;
using System.Text.Json;
using System.Text.RegularExpressions;
using Akka.Actor;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Instance;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.ScriptExecution;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using AuditEvent = ZB.MOM.WW.Audit.AuditEvent;
using ZB.MOM.WW.ScadaBridge.SiteEventLogging;
using ZB.MOM.WW.ScadaBridge.StoreAndForward;
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
///
/// Script Runtime API — injected into Script/Alarm Execution Actors.
/// Provides the API surface that user scripts interact with:
/// Instance.GetAttribute("name")
/// Instance.SetAttribute("name", value)
/// Instance.CallScript("scriptName", params)
/// Scripts.CallShared("scriptName", params)
///
/// (Phase 7): Integration surface APIs:
/// ExternalSystem.Call("systemName", "methodName", params)
/// ExternalSystem.CachedCall("systemName", "methodName", params)
/// Database.Connection("name")
/// Database.CachedWrite("name", "sql", params)
/// Notify.To("listName").Send("subject", "message")
///
/// Recursion Limit — call depth tracked and enforced.
///
public class ScriptRuntimeContext
{
private readonly IActorRef _instanceActor;
private readonly SharedScriptLibrary _sharedScriptLibrary;
private readonly int _currentCallDepth;
private readonly int _maxCallDepth;
private readonly TimeSpan _askTimeout;
///
/// The logger every runtime diagnostic in this context goes through. NOT readonly because a
/// TRACED run swaps in a in the constructor (step-logs
/// design 2026-09-09 D6), so the warnings the runtime logs on the script's behalf become
/// ScriptLog rows attributed to the step that was running. An untraced run keeps the
/// injected logger untouched — no decorator, no allocation.
///
private ILogger _logger;
///
/// The UNDECORATED injected logger. Every diagnostic emitted from inside the log path itself
/// () must use this one: routing it through would be
/// captured by the decorator and re-enter in a loop. Also the logger a
/// child (inline shared-script) context is built with, since
/// refuses to wrap itself.
///
private readonly ILogger _innerLogger;
private readonly string _instanceName;
///
/// WaitForAttribute (spec §4.3): the per-script execution-timeout token from
/// the owning script or alarm on-trigger run
/// (cts.Token). Bounds the Attributes.WaitAsync Ask so a script
/// that hits its own ExecutionTimeoutSeconds abandons the wait. Defaults
/// to for contexts that do not thread one
/// (legacy callers / tests / the alarm path when it has no CTS).
///
private readonly CancellationToken _scriptTimeoutToken;
///
/// External system client for ExternalSystem.Call/CachedCall.
///
private readonly IExternalSystemClient? _externalSystemClient;
///
/// Database gateway for Database.Connection/CachedWrite.
///
private readonly IDatabaseGateway? _databaseGateway;
///
/// Notification Outbox: the site Store-and-Forward Engine that Notify.Send
/// enqueues notifications into. The S&F engine forwards them to central.
///
private readonly StoreAndForwardService? _storeAndForward;
///
/// Notification Outbox: the site communication actor that Notify.Status
/// queries central through (via the gRPC command/control transport).
///
private readonly ICanTell? _siteCommunicationActor;
///
/// Notification Outbox: this site's identifier, stamped on enqueued notifications.
///
private readonly string _siteId;
///
/// SourceNode-stamping: the cluster node name supplied by
/// INodeIdentityProvider on the local host — node-a/node-b
/// for site nodes. Stamped onto NotificationSubmit.SourceNode by
/// and onto SiteCallOperational.SourceNode
/// by the four /
/// cached-call telemetry construction sites so central can persist it on the
/// Notifications / SiteCalls rows. Null when no provider is
/// wired (legacy hosts / tests) — the helper construction sites pass null
/// through verbatim, leaving the central row's SourceNode as NULL too.
///
private readonly string? _sourceNode;
///
/// Notification Outbox (FU3): identifier of the script currently executing in this
/// context — stamped onto NotificationSubmit.SourceScript for the central
/// audit trail. Uses the Site Event Logging "Source" convention
/// ("ScriptActor:<scriptName>"). Null when no single script owns the
/// context (e.g. alarm on-trigger paths that do not wire the Notify outbox).
///
private readonly string? _sourceScript;
///
/// Site event logger for recording recursion-limit violations
/// to the local SQLite event log. Optional — when null the emission is
/// skipped; the existing _logger.LogError + throw path is unchanged.
///
private readonly ISiteEventLogger? _siteEventLogger;
///
/// Best-effort emitter for boundary-crossing actions executed
/// by the script. Optional — when null the helpers degrade to a no-op audit
/// path so tests / contexts that do not need the audit pipeline still work.
///
private readonly IAuditWriter? _auditWriter;
///
/// Site-local tracking store consulted by
/// Tracking.Status(TrackedOperationId). Optional — when null the
/// helper throws on access, mirroring the existing
/// "service-not-wired" behaviour of the other integration helpers.
///
private readonly IOperationTrackingStore? _operationTrackingStore;
///
/// Site-side dual emitter for
/// cached-call lifecycle telemetry. Optional — when null
/// ExternalSystem.CachedCall / Database.CachedWrite still
/// return a and invoke the underlying
/// store-and-forward path, but no audit / SiteCalls telemetry is emitted
/// (tests / minimal hosts that don't wire the audit pipeline).
///
private readonly ICachedCallTelemetryForwarder? _cachedForwarder;
///
/// The per-execution id for this script run. Every
/// trust-boundary audit row emitted by this script execution
/// (sync ApiCall/DbWrite, cached-call lifecycle rows,
/// NotifySend) is stamped into AuditEvent.ExecutionId with
/// this value so all the rows from one script run can be correlated
/// together — independently of the per-operation
/// AuditEvent.CorrelationId.
///
private readonly Guid _executionId;
///
/// (ParentExecutionId): the spawning execution's
/// when this script run was spawned by another
/// execution — the inbound request's per-request execution id for an
/// inbound-API-routed call, the caller's id for a nested
/// CallScript/CallShared, and the writing execution's id for an
/// alarm on-trigger run fired by a script- or inbound-API-initiated attribute
/// write. null for genuinely top-level runs: timer-triggered scripts,
/// script value-change triggers, and alarms fired by DCL (external) data. The
/// spawned script still mints its OWN fresh ; this
/// field records the spawner so a spawned execution's audit rows can point
/// back at the execution that spawned it.
///
private readonly Guid? _parentExecutionId;
///
/// Flow Debugger (design 2026-09-05 §5.2): true when this run is traced — the
/// Trace flag rode in beside ParentExecutionId on the spawning message
/// (or ApiMethod.TraceAllCalls set it at the root). Gates every
/// ScriptTrace row this context emits and is inherited by nested runs.
///
private readonly bool _isTraced;
///
/// Flow Debugger (design §5.4): the per-run inline sink the launcher attaches to
/// ScriptCallResult.TraceEvents. Null when untraced.
///
private readonly FlowTraceCollector? _traceCollector;
///
/// Outbound-calls addendum (plan 2026-09-06 D5): as a delegate the
/// ExternalSystem / Database / Notify helpers hand every
/// ApiOutbound / DbOutbound / Notification row to — the same
/// instance the audit writer / telemetry forwarder receives, so
/// a Live run's inline reply carries them. Null when untraced so the helpers' emit paths
/// stay byte-identical to the pre-feature behaviour.
///
private readonly Action? _traceSink;
///
/// Step logs (design 2026-09-09 D7): the per-run ScriptLog row cap this context's
/// enforces, carried over verbatim into an inline shared-script child.
///
private readonly int _maxLogLinesPerRun;
///
/// Step logs (design D4): the bare script name a ScriptLog row targets — the SAME value
/// the run's ScriptRun anchor row uses, so the Flow Debugger's anchor fallback lines up.
/// Derived from _sourceScript by dropping its ScriptActor: / AlarmActor:
/// prefix; the instance name when no script owns the context.
///
private readonly string _logTarget;
///
/// Step logs (design D4): per-run emission counter stamped on every ScriptLog row as
/// ScriptTraceExtra.Seq, so lines produced inside the same millisecond still sort.
///
private int _logSeq;
///
/// Step logs (design 2026-09-09 D4/D5): the Log global site scripts call —
/// Log.Debug/Info/Warn/Error(message, exception?).
///
/// Every line ALWAYS reaches the node's , traced or not. A
/// ScriptLog trace row is additionally produced only when this run is traced: an
/// untraced context builds the logger with a null emitter, so the row path is one null check
/// and the untraced run is byte-identical to the pre-feature behaviour.
///
/// Bounded per run (D7) and never throws into the script — see
/// .
///
/// An inline shared script SHARES its caller's logger (design §5): a context built
/// by is handed this same instance rather than
/// making its own, so the caller and every shared script it calls share ONE per-run line cap and
/// ONE closing "dropped lines" row. The consequence to know: a shared script's lines carry the
/// PARENT run's ExecutionId and therefore land on the parent's ScriptRun node in
/// the Flow Debugger — not on the shared script's own execution node, even though its
/// ApiCall/DbWrite/step rows still chain under their own execution.
///
public ScriptLogger Log { get; }
///
/// Initializes a new script runtime context with all dependencies for script
/// execution, external system calls, database access, and notification delivery.
///
/// Reference to the Instance Actor managing this instance's state.
/// Library containing shared scripts available to all instances.
/// Current recursion depth of script calls.
/// Maximum allowed recursion depth before an error is thrown.
/// Timeout for Ask pattern messages to the Instance Actor.
/// Unique name of the instance being executed.
/// Logger for diagnostics and warnings.
/// Optional client for external system API calls.
/// Optional gateway for database connection and cached write access.
/// Optional store-and-forward service for notification delivery.
/// Optional actor for site-to-central communication.
/// Identifier of the site where this instance is running.
/// Optional name of the source script for audit trail identification.
/// Optional writer for audit log entries.
/// Optional store for tracking cached operation status.
/// Optional forwarder for cached call telemetry.
///
/// The per-execution id for this script run. When omitted
/// (tag-change / timer-triggered executions) a fresh id is generated; an
/// inbound caller may supply one to tie the execution to an upstream
/// request. Stamped into AuditEvent.ExecutionId on every
/// trust-boundary audit row this execution emits.
///
///
/// (ParentExecutionId): the spawning execution's
/// ExecutionId — supplied for an inbound-API-routed call (the
/// inbound request's per-request id), null for normal (tag-change /
/// timer-triggered) runs. The routed script still generates its own fresh
/// ; this only records the spawner.
///
/// Optional cluster node identifier (node-a/node-b) for audit trail stamping.
///
/// Optional site event logger. When supplied, recursion-limit
/// violations at CallScript and CallShared emit a
/// script Error event in addition to the existing
/// ILogger.LogError + throw. When null the existing behaviour is
/// unchanged; all existing callers and tests remain source-compatible.
///
///
/// WaitForAttribute (spec §4.3): the per-script execution-timeout token
/// (cts.Token on the owning execution actor) used to bound
/// Attributes.WaitAsync. Defaults to
/// for callers / tests that do not
/// thread one — those waits are bounded only by their own timeout.
///
/// Flow Debugger trace flag — see design 2026-09-05 §5.2. Defaults to untraced.
/// Flow Debugger per-run inline trace sink (design §5.4); null when untraced.
///
/// Step logs (design 2026-09-09 D7): the per-run ScriptLog row cap for
/// — mirrors SiteRuntimeOptions.MaxLogLinesPerRun, which the script run launcher passes
/// in. Values below 1 are clamped to 1: a mis-set option must not make every script run throw.
///
///
/// Step logs (design §5): an existing to adopt as
/// instead of creating one — supplied only by
/// , so an inline shared script shares its
/// caller's line cap and anchor. Null (the default) creates this run's own logger.
///
public ScriptRuntimeContext(
IActorRef instanceActor,
SharedScriptLibrary sharedScriptLibrary,
int currentCallDepth,
int maxCallDepth,
TimeSpan askTimeout,
string instanceName,
ILogger logger,
IExternalSystemClient? externalSystemClient = null,
IDatabaseGateway? databaseGateway = null,
StoreAndForwardService? storeAndForward = null,
ICanTell? siteCommunicationActor = null,
string siteId = "",
string? sourceScript = null,
IAuditWriter? auditWriter = null,
IOperationTrackingStore? operationTrackingStore = null,
ICachedCallTelemetryForwarder? cachedForwarder = null,
Guid? executionId = null,
Guid? parentExecutionId = null,
string? sourceNode = null,
ISiteEventLogger? siteEventLogger = null,
CancellationToken scriptTimeoutToken = default,
bool isTraced = false,
FlowTraceCollector? traceCollector = null,
int maxLogLinesPerRun = 200,
ScriptLogger? sharedLog = null)
{
_instanceActor = instanceActor;
_sharedScriptLibrary = sharedScriptLibrary;
_currentCallDepth = currentCallDepth;
_maxCallDepth = maxCallDepth;
_askTimeout = askTimeout;
_instanceName = instanceName;
_innerLogger = logger;
_logger = logger;
_externalSystemClient = externalSystemClient;
_databaseGateway = databaseGateway;
_storeAndForward = storeAndForward;
_siteCommunicationActor = siteCommunicationActor;
_siteId = siteId;
_sourceScript = sourceScript;
_auditWriter = auditWriter;
_operationTrackingStore = operationTrackingStore;
_cachedForwarder = cachedForwarder;
// SourceNode-stamping: the local node name read from
// INodeIdentityProvider at the launching script run; null when no
// provider was wired so the downstream callsites pass null through
// verbatim — leaving central SourceNode as NULL.
_sourceNode = sourceNode;
_executionId = executionId ?? Guid.NewGuid();
// (ParentExecutionId): stored verbatim — no `?? NewGuid()`
// fallback. A non-routed run legitimately has no parent and stays null.
_parentExecutionId = parentExecutionId;
// Optional — null when not wired (tests / alarm on-trigger runs).
_siteEventLogger = siteEventLogger;
// WaitForAttribute (spec §4.3): default(CancellationToken) == None when
// not threaded in — the WaitAsync Ask is then bounded only by its own timeout.
_scriptTimeoutToken = scriptTimeoutToken;
_isTraced = isTraced;
_traceCollector = traceCollector;
_traceSink = traceCollector is null ? null : Collect;
// Step logs (design 2026-09-09 D4-D6). ORDER MATTERS: the Log global is built over the
// UNDECORATED logger first, and only then — when traced — does the context's own logger
// become the capturing decorator over that same sink. Building them the other way round
// would hand ScriptLogger the decorator (every Log.* line logged, captured and re-emitted
// as a second row) and TraceCapturingLogger refuses to wrap itself for exactly that reason.
_maxLogLinesPerRun = Math.Max(1, maxLogLinesPerRun);
_logTarget = string.IsNullOrEmpty(_sourceScript)
// No single script owns the context — the instance is the closest honest target.
? _instanceName
// "ScriptActor:Foo" / "AlarmActor:Foo" -> "Foo"; a bare name is unchanged (IndexOf -1).
: _sourceScript[(_sourceScript.IndexOf(':') + 1)..];
// An inline shared script adopts its caller's logger (design §5) — one counter, one anchor,
// one closing row for the whole inline chain. Everyone else builds their own.
Log = sharedLog ?? new ScriptLogger(
logger, isTraced ? EmitLog : null, _maxLogLinesPerRun, _instanceName, _logTarget, _executionId);
if (isTraced)
{
// The child's OWN undecorated logger, decorated over the SHARED sink: the capturing
// decorator is per-context (it must not be handed on), the sink behind it is not.
_logger = new TraceCapturingLogger(logger, Log);
}
}
///
/// Flow Debugger: the single feeder of the inline collector. Every row this run's
/// reply should carry — the ScriptTrace rows from and the
/// outbound rows the helpers emit — passes through here; a no-op when untraced or once
/// the launcher has d the run.
///
private void Collect(AuditEvent evt)
{
try
{
_traceCollector?.Add(evt);
}
catch (Exception ex)
{
// The collector is a debug-only observer: it must never fail a script call, and it
// must never keep the durable audit row from reaching the writer (review, 2026-09-06).
_logger.LogWarning(ex, "Flow trace collector rejected {Action} ({EventId})", evt.Action, evt.EventId);
}
}
///
/// Wraps a trace sink so a throwing observer can neither fail the script call nor skip the
/// durable audit write that follows it. Null in, null out — an untraced run allocates nothing.
///
internal static Action? GuardTraceSink(Action? sink, ILogger logger)
{
if (sink is null) return null;
return evt =>
{
try
{
sink(evt);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Flow trace sink rejected {Action} ({EventId})", evt.Action, evt.EventId);
}
};
}
/// Flow Debugger (design 2026-09-05 §5.2): whether this run emits ScriptTrace rows.
public bool IsTraced => _isTraced;
///
/// This run's own per-execution id. Exposed so a
/// nested Scripts.CallShared can record it as the spawned shared
/// script's ParentExecutionId, forming a true execution tree.
///
internal Guid ExecutionId => _executionId;
///
/// The spawning execution's id for this run (null for
/// a root run). Exposed for test assertions on the execution tree.
///
internal Guid? ParentExecutionId => _parentExecutionId;
///
/// (ParentExecutionId tag-cascade): builds a child
/// for an inline Scripts.CallShared
/// invocation. The shared script runs inline (no actor hop) but is modelled
/// as its OWN execution node in the audit tree: it mints a fresh
/// and records THIS run's
/// as its ParentExecutionId, so B → CallShared(C) yields
/// C.ParentExecutionId == B.ExecutionId. Every other dependency
/// (actors, gateways, audit writer, site id, source node, call-depth) is
/// carried over verbatim from this context.
///
/// The recursion depth of the shared-script call.
/// A new child for the shared-script invocation.
internal ScriptRuntimeContext CreateChildContextForSharedScript(int childCallDepth)
{
return new ScriptRuntimeContext(
_instanceActor,
_sharedScriptLibrary,
childCallDepth,
_maxCallDepth,
_askTimeout,
_instanceName,
// Step logs (D6): the UNDECORATED logger — the child installs its own capturing
// decorator when traced, and TraceCapturingLogger refuses to wrap another one.
_innerLogger,
_externalSystemClient,
_databaseGateway,
_storeAndForward,
_siteCommunicationActor,
_siteId,
_sourceScript,
_auditWriter,
_operationTrackingStore,
_cachedForwarder,
// Fresh execution id for the shared-script run (omit so the ctor mints one)…
executionId: null,
// …parented to THIS run's execution id (the spawner).
parentExecutionId: _executionId,
sourceNode: _sourceNode,
siteEventLogger: _siteEventLogger,
// WaitForAttribute (spec §4.3): an inline shared-script call shares the
// parent run's execution-timeout token so a WaitAsync inside the shared
// script is bounded by the SAME script deadline.
scriptTimeoutToken: _scriptTimeoutToken,
// Flow Debugger (design §5.2): a traced root traces its whole tree — the inline
// shared script shares the same flag AND the same inline sink.
isTraced: _isTraced,
traceCollector: _traceCollector,
maxLogLinesPerRun: _maxLogLinesPerRun,
// Step logs (design §5): the inline shared script is its own execution node for step
// rows, but it SHARES this run's Log — one line cap and one anchor for the whole
// inline chain, so its lines carry this run's ExecutionId.
sharedLog: Log);
}
///
/// Flow Debugger (design 2026-09-05 §5.1): emits one ScriptTrace audit row for a
/// traced run — into the inline collector first (so a routed call can return it) and
/// then through the best-effort . No-op when untraced, never
/// throws into the script, and hands raw payloads to the writer whose redactor applies
/// the configured size cap (§9) exactly as the other channels do.
///
/// The trace row's kind.
/// The row's lifecycle status.
/// The row's target (attribute name, script name, …).
/// When the traced action started — the row's OccurredAtUtc.
/// How long the action took.
/// Request summary.
/// Response summary.
/// Error message on a failed row.
/// Kind-specific extra detail.
///
/// Step logs (design 2026-09-09 D6): the step's PRE-MINTED EventId, so the log lines the
/// runtime produced inside the step's can name it. Null mints one.
///
private void EmitTrace(
AuditKind kind, AuditStatus status, string target, DateTime startedUtc, int durationMs,
string? input, string? output, string? error, ScriptTraceExtra? extra, Guid? eventId = null)
{
if (!_isTraced) return;
AuditEvent evt;
try
{
evt = ScadaBridgeAuditEventFactory.Create(
AuditChannel.ScriptTrace, kind, status,
eventId: eventId,
occurredAtUtc: DateTime.SpecifyKind(startedUtc, DateTimeKind.Utc),
actor: _sourceScript, target: target, sourceNode: _sourceNode,
executionId: _executionId, parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName, sourceScript: _sourceScript,
durationMs: durationMs, errorMessage: error,
requestSummary: input, responseSummary: output,
extra: extra?.ToJson());
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Flow trace event build failed for {Kind}", kind);
return;
}
Collect(evt);
if (_auditWriter is null) return;
try
{
var t = _auditWriter.WriteAsync(evt, CancellationToken.None);
if (!t.IsCompletedSuccessfully)
{
_ = t.ContinueWith(
x => _logger.LogWarning(x.Exception, "Flow trace write failed for {Kind}", kind),
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Flow trace write failed for {Kind}", kind);
}
}
///
/// Step logs (design 2026-09-09 D4/D7): emits one ScriptTrace/ScriptLog row for a
/// single log line — the collector's LOW-PRIORITY lane first (log lines must never crowd out
/// the step rows a run emits at its end), then the best-effort .
///
/// This is the emitter handed to , and it is wired only when the run is
/// traced. Every diagnostic it logs itself goes to , NEVER to
/// : on a traced run the latter is the capturing decorator, which would
/// feed the warning straight back into this method.
///
/// Level name as it appears on the wire: debug|info|warn|error.
/// Line origin: script (a Log.* call) or runtime.
/// The already-clipped message text.
/// The exception attached to the line, if any.
/// The owning step's EventId; null falls back to the run's anchor row (D2).
private void EmitLog(string level, string source, string message, Exception? exception, Guid? stepEventId)
{
if (!_isTraced) return;
AuditEvent evt;
try
{
evt = ScadaBridgeAuditEventFactory.Create(
AuditChannel.ScriptTrace, AuditKind.ScriptLog, AuditStatus.Delivered,
occurredAtUtc: DateTime.UtcNow,
actor: _sourceScript, target: _logTarget, sourceNode: _sourceNode,
executionId: _executionId, parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName, sourceScript: _sourceScript,
errorMessage: exception?.Message, errorDetail: exception?.ToString(),
requestSummary: message,
extra: new ScriptTraceExtra
{
LogLevel = level,
LogSource = source,
StepEventId = stepEventId,
Seq = Interlocked.Increment(ref _logSeq),
}.ToJson());
}
catch (Exception ex)
{
_innerLogger.LogWarning(ex, "Flow trace log-line build failed at {Level}", level);
return;
}
try
{
// Low-priority lane (D7): dropped lines set LogsTruncated, not Truncated.
_traceCollector?.Add(evt, lowPriority: true);
}
catch (Exception ex)
{
_innerLogger.LogWarning(ex, "Flow trace collector rejected log line ({EventId})", evt.EventId);
}
if (_auditWriter is null) return;
try
{
var t = _auditWriter.WriteAsync(evt, CancellationToken.None);
if (!t.IsCompletedSuccessfully)
{
_ = t.ContinueWith(
x => _innerLogger.LogWarning(x.Exception, "Flow trace log-line write failed ({EventId})", evt.EventId),
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
}
catch (Exception ex)
{
_innerLogger.LogWarning(ex, "Flow trace log-line write failed ({EventId})", evt.EventId);
}
}
/// Serialises a small trace payload; null on failure (best-effort, like the audit helpers).
private static string? TraceJson(object? value)
{
if (value is null) return null;
try { return JsonSerializer.Serialize(value, ScriptTraceExtra.JsonOptions); }
catch (Exception) { return null; }
}
///
/// Fire-and-forget emission of a script Error site event
/// for a recursion-limit violation. Mirrors the call shape used by
/// the run's own catch blocks. A fault from
/// the site-event logger is observed-and-dropped (best-effort) via
/// ContinueWith(OnlyOnFaulted) — it never blocks or faults the
/// _logger.LogError + throw path that follows. A null logger is a no-op.
///
private void EmitRecursionLimitEventAsync(string msg)
{
if (_siteEventLogger == null)
return;
var source = string.IsNullOrEmpty(_instanceName)
? "recursion-guard"
: $"InstanceScript:{_instanceName}";
var logTask = _siteEventLogger.LogEventAsync("script", "Error", _instanceName, source, msg);
if (!logTask.IsCompleted)
{
logTask.ContinueWith(
t => _logger.LogWarning(t.Exception,
"Site event log write failed for recursion-limit violation on instance '{Instance}'",
_instanceName),
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
else if (logTask.IsFaulted)
{
_logger.LogWarning(logTask.Exception,
"Site event log write failed for recursion-limit violation on instance '{Instance}'",
_instanceName);
}
}
///
/// Gets the current value of an attribute from the Instance Actor.
/// Uses Ask pattern (system boundary between script execution and instance state).
///
/// Name of the attribute to retrieve.
/// The current attribute value, or null if not found.
public async Task GetAttribute(string attributeName)
{
var correlationId = Guid.NewGuid().ToString();
var request = new GetAttributeRequest(
correlationId, _instanceName, attributeName, DateTimeOffset.UtcNow);
var response = await _instanceActor.Ask(request, _askTimeout);
if (!response.Found)
{
_logger.LogWarning(
"GetAttribute: attribute '{Attribute}' not found on instance '{Instance}'",
attributeName, _instanceName);
}
return response.Value;
}
///
/// WaitForAttribute (spec §3-§5): waits event-driven for an attribute to reach
/// a value (encoded-equality), satisfy a site-local predicate, or change at all,
/// bounded by . Returns true if matched within
/// the timeout, false on timeout — NEVER throws on timeout. The backing
/// Attributes.WaitAsync for the accessor.
///
///
/// The Ask is bounded by the script's own execution-timeout token (§4.3): a
/// script that hits its ExecutionTimeoutSeconds abandons the wait. The
/// Ask timeout is the wait timeout plus a small slack
/// so the InstanceActor's own scheduled timeout reply is the authoritative path
/// for the false/timed-out outcome, not the Ask deadline.
///
///
///
/// Quality-agnostic by default (spec §4.2): a value arriving at Bad
/// quality still satisfies the wait — the match tests the value, not the quality.
/// Opt into a quality-gated ("Good"-only) wait via the requireGoodQuality
/// argument (spec §4.2); the InstanceActor then holds the wait until the value
/// satisfies the test at Good quality (or times out).
///
///
///
/// Never throws on timeout. An
/// (the pathological case where the InstanceActor's authoritative timeout reply
/// never arrives — actor stopped/restarted) is caught and surfaced as false,
/// matching the timeout contract. An /
/// from the script-deadline token is NOT caught
/// — it propagates to abort the script (intended §4.3 behaviour).
///
///
/// The scope-resolved attribute name to wait on.
///
/// The codec-encoded target value; null (with null )
/// means "any change".
///
/// Site-local predicate; null when the encoded target is used.
/// How long to wait before returning false.
///
/// Quality-gated ("Good"-only) mode (spec §4.2): when , a
/// value reaching the target / satisfying the predicate at Bad/Uncertain quality
/// is NOT a match — the wait holds until the value satisfies the test at Good
/// quality (or times out). Defaults to (quality-agnostic).
///
/// true on match within the timeout; false on timeout.
public async Task WaitAttribute(
string name, string? targetValueEncoded, Func? predicate, TimeSpan timeout,
bool requireGoodQuality = false)
=> (await WaitInternal(name, targetValueEncoded, predicate, timeout, requireGoodQuality)).Matched;
///
/// WaitForAttribute (spec §3): the richer overload backing Attributes.WaitForAsync
/// — identical semantics to but surfaces the full
/// (matched flag + matched value + quality + timed-out
/// flag) instead of a bare bool. Never throws on timeout (see ).
///
/// The scope-resolved attribute name to wait on.
/// The codec-encoded target value; null (with null predicate) means "any change".
/// Site-local predicate; null when the encoded target is used.
/// How long to wait before returning a timed-out result.
/// Quality-gated ("Good"-only) mode (spec §4.2); defaults to .
/// The full — on timeout: Matched:false, TimedOut:true.
public async Task WaitAttributeFull(
string name, string? targetValueEncoded, Func? predicate, TimeSpan timeout,
bool requireGoodQuality = false)
{
var r = await WaitInternal(name, targetValueEncoded, predicate, timeout, requireGoodQuality);
return new WaitResult(r.Matched, r.Value, r.Quality, r.TimedOut);
}
///
/// Shared core for / :
/// builds the (incl. the §4.2
/// flag), Asks the InstanceActor bounded by
/// the script's execution-timeout token, and returns the full response. An
/// (the pathological case where the actor's own
/// authoritative timeout reply never arrives — actor stopped/restarted) is caught
/// and surfaced as a synthetic non-matched/timed-out response, preserving the
/// "never throw on timeout" contract. An /
/// from the script-deadline token is NOT caught
/// — it propagates to abort the script (§4.3).
///
private async Task WaitInternal(
string name, string? targetValueEncoded, Func? predicate, TimeSpan timeout,
bool requireGoodQuality)
{
var cid = Guid.NewGuid().ToString();
var req = new WaitForAttributeRequest(
cid, _instanceName, name, targetValueEncoded, predicate, timeout, DateTimeOffset.UtcNow,
requireGoodQuality);
var startedUtc = DateTime.UtcNow;
var sw = _isTraced ? Stopwatch.StartNew() : null;
// Step logs (design D6): pre-mint this step's row id and make it the ambient step for the
// body, so a warning the runtime logs while the wait is in flight names THIS row. Nothing
// is minted and no scope is opened on an untraced run.
Guid? stepId = _isTraced ? Guid.NewGuid() : null;
using var step = stepId is { } waitStepId ? TraceStepScope.Enter(waitStepId) : null;
WaitForAttributeResponse response;
try
{
response = await _instanceActor.Ask(
req, timeout + _askTimeout, _scriptTimeoutToken);
}
catch (AskTimeoutException)
{
// Pathological: the InstanceActor's own scheduled timeout reply never
// arrived (e.g. the actor stopped/restarted under us). The helper's
// contract is "false on timeout, never throw" — so synthesize a
// non-matched/timed-out response rather than leaking the Ask exception.
// OperationCanceledException / TaskCanceledException from the
// script-deadline token are deliberately NOT caught here: they must
// propagate to abort the script (§4.3).
response = new WaitForAttributeResponse(
cid, Matched: false, null, null, TimedOut: true);
}
catch (OperationCanceledException)
{
// Script deadline hit mid-wait: record the cancelled wait for a traced run, then
// let the cancellation propagate exactly as before (§4.3).
if (sw is not null)
{
EmitTrace(AuditKind.AttributeWait, AuditStatus.Failed, name, startedUtc,
(int)sw.ElapsedMilliseconds, null, null, "Wait cancelled by the script deadline",
new ScriptTraceExtra
{
WaitAttribute = name, WaitTarget = targetValueEncoded ?? (predicate is null ? null : ""),
WaitTimeoutMs = (int)timeout.TotalMilliseconds, WaitOutcome = "cancelled",
},
eventId: stepId);
}
throw;
}
if (sw is not null)
{
var outcome = response.Matched ? "satisfied" : response.TimedOut ? "timeout" : "error";
var observed = AttributeValueCodec.Encode(response.Value);
EmitTrace(AuditKind.AttributeWait,
response.Matched ? AuditStatus.Delivered : AuditStatus.Failed,
name, startedUtc, (int)sw.ElapsedMilliseconds,
input: TraceJson(new { attribute = name, target = targetValueEncoded, timeoutMs = (int)timeout.TotalMilliseconds, requireGoodQuality }),
output: TraceJson(new { matched = response.Matched, timedOut = response.TimedOut, value = observed, quality = response.Quality }),
error: response.Matched ? null : response.ErrorMessage ?? (response.TimedOut ? $"Wait for '{name}' timed out after {timeout.TotalSeconds}s" : null),
extra: new ScriptTraceExtra
{
WaitAttribute = name,
WaitTarget = targetValueEncoded ?? (predicate is null ? null : ""),
WaitTimeoutMs = (int)timeout.TotalMilliseconds,
WaitOutcome = outcome,
ObservedValue = observed,
},
eventId: stepId);
}
return response;
}
///
/// Sets an attribute value. For data-connected attributes the Instance Actor
/// forwards the write to the DCL, which writes the physical device; the
/// in-memory value is not optimistically updated. For static attributes the
/// Instance Actor updates the in-memory value and persists the override to
/// SQLite. All mutations are serialized through the Instance Actor mailbox.
///
/// The write is awaited so that a device-write failure on a data-connected
/// attribute is surfaced synchronously to the calling script as an
/// .
///
/// Name of the attribute to set.
/// String value to set for the attribute.
/// A task that represents the asynchronous operation.
public async Task SetAttribute(string attributeName, string value)
{
var correlationId = Guid.NewGuid().ToString();
var command = new SetStaticAttributeCommand(
correlationId, _instanceName, attributeName, value, DateTimeOffset.UtcNow,
// (ParentExecutionId tag-cascade): stamp THIS run's own
// ExecutionId as the write's originating execution. The Instance Actor
// carries it onto the published AttributeValueChanged, so an alarm this
// write trips records this run as the on-trigger script's parent.
SourceExecutionId: _executionId,
// Flow Debugger (design §5.2/§5.3): a traced run's write arms the instance
// actor's echo stamp and rides onto the published AttributeValueChanged.
Trace: _isTraced);
var startedUtc = DateTime.UtcNow;
var sw = _isTraced ? Stopwatch.StartNew() : null;
// Step logs (design D6): one pre-minted row id shared by BOTH emit points below, opened as
// the ambient step so runtime warnings logged during the write are attributed to it.
Guid? stepId = _isTraced ? Guid.NewGuid() : null;
using var step = stepId is { } writeStepId ? TraceStepScope.Enter(writeStepId) : null;
SetStaticAttributeResponse response;
try
{
// Ask — mutation serialized through the Instance Actor mailbox; the reply
// carries the device-write outcome for data-connected attributes.
response = await _instanceActor.Ask(command, _askTimeout);
}
catch (Exception ex) when (sw is not null)
{
EmitTrace(AuditKind.AttributeWrite, AuditStatus.Failed, attributeName, startedUtc,
(int)sw.ElapsedMilliseconds, TraceJson(new { attribute = attributeName, value }), null, ex.Message,
new ScriptTraceExtra { Writes = [new AttributeWriteDetail(attributeName, value, false, ex.Message)] },
eventId: stepId);
throw;
}
if (sw is not null)
{
EmitTrace(AuditKind.AttributeWrite,
response.Success ? AuditStatus.Delivered : AuditStatus.Failed,
attributeName, startedUtc, (int)sw.ElapsedMilliseconds,
input: TraceJson(new { attribute = attributeName, value }),
output: null,
error: response.ErrorMessage,
extra: new ScriptTraceExtra
{
Writes = [new AttributeWriteDetail(attributeName, value, response.Success, response.ErrorMessage)],
},
eventId: stepId);
}
if (!response.Success)
{
throw new InvalidOperationException(
$"SetAttribute('{attributeName}') failed: {response.ErrorMessage}");
}
}
///
/// Writes a SET of data-sourced attributes to the device in ONE DCL batch round-trip,
/// then writes a trigger flag, then waits EVENT-DRIVEN (reusing the existing
/// waiter — not a poll) for
/// to reach , bounded by .
/// Replaces N sequential per-attribute writes with a single gateway call before the
/// wait. Throws if the write/trigger leg fails
/// (resolution error, multi-connection batch, device error); returns the wait result
/// otherwise (true = matched, false = timeout, never throws on timeout).
///
/// Scope-resolved attribute name → codec-encoded value to batch-write.
/// Scope-resolved trigger attribute name written AFTER the batch.
/// Codec-encoded value for the trigger.
/// Scope-resolved attribute to wait on.
/// Codec-encoded target value (null ⇒ any change).
/// How long to wait for the response before returning false.
/// true on response match within the timeout; false on timeout.
public async Task WriteBatchAndWait(
IReadOnlyDictionary encodedValues, string triggerAttr, string? triggerEncoded,
string responseAttr, string? responseEncoded, TimeSpan timeout)
{
var cid = Guid.NewGuid().ToString();
var batchReq = new WriteAttributeBatchRequest(
cid, _instanceName, encodedValues, triggerAttr, triggerEncoded, DateTimeOffset.UtcNow);
// 35s: the InstanceActor's DCL Ask is internally bounded at 30s, so allow a small
// margin so the DCL's own typed failure/timeout reply is the one we observe rather
// than an AskTimeoutException here. Honors the script execution-timeout token.
var startedUtc = DateTime.UtcNow;
var sw = _isTraced ? Stopwatch.StartNew() : null;
// Step logs (design D6): the batch's own step. The trailing WaitAttribute opens its own
// nested scope and restores this one on the way out, so each leg owns its lines.
Guid? stepId = _isTraced ? Guid.NewGuid() : null;
using var step = stepId is { } batchStepId ? TraceStepScope.Enter(batchStepId) : null;
var batchResp = await _instanceActor.Ask(
batchReq, TimeSpan.FromSeconds(35), _scriptTimeoutToken);
if (sw is not null)
{
// Flow Debugger (design §5.1): one AttributeWrite row per batch — every value plus the
// trigger — with the batch's single outcome on each write.
var writes = encodedValues
.Select(kv => new AttributeWriteDetail(kv.Key, kv.Value, batchResp.Success, batchResp.ErrorMessage))
.Append(new AttributeWriteDetail(triggerAttr, triggerEncoded, batchResp.Success, batchResp.ErrorMessage))
.ToList();
EmitTrace(AuditKind.AttributeWrite,
batchResp.Success ? AuditStatus.Delivered : AuditStatus.Failed,
triggerAttr, startedUtc, (int)sw.ElapsedMilliseconds,
input: TraceJson(new { values = encodedValues, trigger = triggerAttr, triggerValue = triggerEncoded }),
output: null,
error: batchResp.ErrorMessage,
extra: new ScriptTraceExtra { TriggerAttribute = triggerAttr, TriggerValue = triggerEncoded, Writes = writes },
eventId: stepId);
}
if (!batchResp.Success)
throw new InvalidOperationException(
$"WriteBatchAndWait write failed: {batchResp.ErrorMessage}");
return await WaitAttribute(responseAttr, responseEncoded, null, timeout);
}
///
/// Calls a sibling script on the same instance by name (Ask pattern).
/// Enforces recursion limit.
/// Uses Ask pattern for CallScript.
/// may be a dictionary or an anonymous object
/// (new { name = "Bob" }) — see .
///
/// Name of the script to call.
/// Optional parameters to pass to the script (dictionary or anonymous object).
/// The return value from the called script.
public async Task CallScript(string scriptName, object? parameters = null)
{
var nextDepth = _currentCallDepth + 1;
if (nextDepth > _maxCallDepth)
{
var msg = $"Script call depth exceeded maximum of {_maxCallDepth}. " +
$"CallScript('{scriptName}') rejected at depth {nextDepth}.";
_logger.LogError(msg);
// Emit to site event log in addition to ILogger; fire-and-forget.
EmitRecursionLimitEventAsync(msg);
throw new InvalidOperationException(msg);
}
var correlationId = Guid.NewGuid().ToString();
var request = new ScriptCallRequest(
scriptName,
ScriptArgs.Normalize(parameters),
nextDepth,
correlationId,
// (ParentExecutionId tag-cascade): the child
// script run is a NEW execution spawned BY this run. Its parent is
// THIS run's own ExecutionId — NOT the inherited _parentExecutionId.
// So A → CallScript(B) yields B.ParentExecutionId == A.ExecutionId,
// building a true multi-level execution tree rather than flattening
// every nested call under the original inbound spawner.
ParentExecutionId: _executionId,
// Flow Debugger (design §5.2): nested runs inherit the trace flag.
Trace: _isTraced);
// Ask the Instance Actor, which routes to the appropriate Script Actor
var result = await _instanceActor.Ask(request, _askTimeout);
if (!result.Success)
{
throw new InvalidOperationException(
$"CallScript('{scriptName}') failed: {result.ErrorMessage}");
}
return result.ReturnValue;
}
///
/// Provides access to shared script execution via the Scripts property.
///
public ScriptCallHelper Scripts => new(_sharedScriptLibrary, this, _currentCallDepth, _maxCallDepth, _logger);
///
/// Provides access to external system calls.
/// ExternalSystem.Call("systemName", "methodName", params)
/// ExternalSystem.CachedCall("systemName", "methodName", params)
///
public ExternalSystemHelper ExternalSystem => new(
_externalSystemClient, _instanceName, _logger, _executionId, _auditWriter, _siteId, _sourceScript,
// Emit CachedSubmit telemetry
// on every ExternalSystem.CachedCall enqueue.
_cachedForwarder,
// (ParentExecutionId): the spawning execution's id,
// threaded alongside _executionId. Null for non-routed runs.
_parentExecutionId,
// SourceNode-stamping: the local node name (node-a/node-b),
// threaded so the cached-call telemetry construction sites can stamp
// it onto SiteCallOperational.SourceNode.
_sourceNode,
// Flow Debugger: the outbound rows ride the inline collector (null when untraced).
traceSink: _traceSink,
// Step logs (D6): the run's trace flag — the authority for the sync Call's step scope.
isTraced: _isTraced);
///
/// Provides access to database operations.
/// Database.Connection("name")
/// Database.CachedWrite("name", "sql", params)
///
public DatabaseHelper Database => new(
_databaseGateway,
_instanceName,
_logger,
_executionId,
// Wire the IAuditWriter so
// Database.Connection(name) returns an auditing decorator that
// emits one DbOutbound/DbWrite row per script-initiated
// Execute / ExecuteScalar / ExecuteReader.
_auditWriter,
_siteId,
_sourceScript,
// Emit CachedSubmit telemetry on
// every Database.CachedWrite enqueue.
_cachedForwarder,
// (ParentExecutionId): the spawning execution's id,
// threaded alongside _executionId. Null for non-routed runs.
_parentExecutionId,
// SourceNode-stamping: the local node name (node-a/node-b),
// threaded so Database.CachedWrite's CachedSubmit telemetry can
// stamp it onto SiteCallOperational.SourceNode.
_sourceNode,
// Flow Debugger: the outbound rows ride the inline collector (null when untraced).
traceSink: _traceSink);
///
/// Provides access to the Notification Outbox API.
/// Notify.To("listName").Send("subject", "message") enqueues a notification
/// for central delivery and returns its NotificationId;
/// Notify.Status(id) queries the delivery status of that notification.
///
///
/// The is threaded
/// through so Notify.To(list).Send(...) emits one
/// Notification/NotifySend audit row per accepted submission.
/// Best-effort per alog.md §7 — a thrown writer never aborts the script's
/// Send.
///
public NotifyHelper Notify => new(
_storeAndForward, _siteCommunicationActor, _siteId, _instanceName, _sourceScript, _askTimeout, _logger,
_executionId, _auditWriter,
// (ParentExecutionId): the spawning execution's id,
// threaded alongside _executionId. Null for non-routed runs.
_parentExecutionId,
// SourceNode-stamping: the local node name (node-a/node-b),
// threaded so NotifyTarget.Send can stamp it onto NotificationSubmit.
_sourceNode,
// Flow Debugger: the NotifySend row rides the inline collector (null when untraced).
traceSink: _traceSink);
///
/// Read-only access to the instance's CURRENT alarm conditions (MES alarm-status API
/// §5.2). Alarms.CurrentAsync() returns one per alarm
/// the instance holds — computed alarms and mirrored native (OPC UA A&C / MxAccess
/// Gateway) conditions alike — so a Call script can answer "what is in alarm
/// right now?" without an on-trigger Alarm context.
///
public AlarmsAccessor Alarms => new(this);
///
/// Backing read for : Asks THIS instance's
/// Instance Actor for its retained alarm rows and projects them to
/// . The script executes inside its own instance's context, so
/// this is a LOCAL Ask on the same node — the same mechanism as
/// , never a cross-cluster hop.
///
///
/// Cancels the wait for the actor's reply. Defaults to ;
/// the Ask is bounded by the context's ask timeout regardless.
///
/// The instance's current alarm conditions; empty when it holds none.
internal async Task> GetAlarmsAsync(
CancellationToken cancellationToken = default)
{
var request = new GetAlarmSnapshotRequest(
Guid.NewGuid().ToString(), _instanceName, DateTimeOffset.UtcNow);
var response = await _instanceActor.Ask(
request, _askTimeout, cancellationToken);
return response.Alarms.Select(ToScriptAlarm).ToList();
}
///
/// Projects a retained onto the flat, script-facing
/// shape. Condition is the authority for the
/// active/acked/severity fields (it already folds a computed alarm's State + Priority
/// into the same unified shape), so computed and native alarms read identically from a
/// script. Shelved collapses the shelve sub-states to a single boolean — scripts
/// care whether the operator has parked the alarm, not which shelve flavour was used.
///
private static ScriptAlarm ToScriptAlarm(AlarmStateChanged a) => new(
Name: a.AlarmName,
SourceReference: a.SourceReference,
NativeSourceCanonicalName: a.NativeSourceCanonicalName,
Active: a.Condition.Active,
Acknowledged: a.Condition.Acknowledged,
Confirmed: a.Condition.Confirmed,
Shelved: a.Condition.Shelve != AlarmShelveState.Unshelved,
Suppressed: a.Condition.Suppressed,
Severity: a.Condition.Severity,
Kind: a.Kind.ToString(),
Message: a.Message,
AlarmTypeName: a.AlarmTypeName,
Category: a.Category,
OperatorUser: a.OperatorUser,
OperatorComment: a.OperatorComment,
OriginalRaiseTime: a.OriginalRaiseTime,
Timestamp: a.Timestamp,
AckTime: a.AckTime,
CurrentValue: a.CurrentValue,
LimitValue: a.LimitValue,
IsConfiguredPlaceholder: a.IsConfiguredPlaceholder);
///
/// Site-local tracking-status API for cached operations.
/// Tracking.Status(trackedOperationId) reads the site SQLite tracking row
/// directly (authoritative source of truth — no central round-trip) and
/// returns a , or null when the
/// id is unknown / has already been purged.
///
public TrackingHelper Tracking => new(_operationTrackingStore, _logger);
///
/// Helper class for Scripts.CallShared() syntax.
///
///
/// Helper class for calling shared scripts through the Scripts property.
///
public class ScriptCallHelper
{
private readonly SharedScriptLibrary _library;
private readonly ScriptRuntimeContext _context;
private readonly int _currentCallDepth;
private readonly int _maxCallDepth;
private readonly ILogger _logger;
///
/// Initializes a new shared script call helper.
///
/// The shared script library containing available scripts.
/// The runtime context of the calling script.
/// Current recursion depth of script calls.
/// Maximum allowed recursion depth.
/// Logger for diagnostics.
internal ScriptCallHelper(
SharedScriptLibrary library,
ScriptRuntimeContext context,
int currentCallDepth,
int maxCallDepth,
ILogger logger)
{
_library = library;
_context = context;
_currentCallDepth = currentCallDepth;
_maxCallDepth = maxCallDepth;
_logger = logger;
}
///
/// Executes a shared script inline (direct method call, not actor message).
/// Enforces recursion limit.
/// may be a dictionary or an anonymous
/// object (new { name = "Bob" }) — see .
///
/// Name of the shared script to execute.
/// Optional parameters to pass to the script (dictionary or anonymous object).
/// Cancellation token for async execution.
/// The return value from the shared script.
public async Task CallShared(
string scriptName,
object? parameters = null,
CancellationToken cancellationToken = default)
{
var nextDepth = _currentCallDepth + 1;
if (nextDepth > _maxCallDepth)
{
var msg = $"Script call depth exceeded maximum of {_maxCallDepth}. " +
$"CallShared('{scriptName}') rejected at depth {nextDepth}.";
_logger.LogError(msg);
// Emit to site event log via the parent context's
// helper — single emission path, fire-and-forget.
_context.EmitRecursionLimitEventAsync(msg);
throw new InvalidOperationException(msg);
}
// (ParentExecutionId tag-cascade): the shared
// script runs inline, but is modelled as its OWN execution node — a
// child context mints a fresh ExecutionId parented to the caller's
// ExecutionId, so its audit rows chain under the calling run.
var childContext = _context.CreateChildContextForSharedScript(nextDepth);
return await _library.ExecuteAsync(
scriptName, childContext, ScriptArgs.Normalize(parameters), cancellationToken);
}
}
///
/// Helper for ExternalSystem.Call/CachedCall syntax.
///
///
/// Every invocation emits
/// one ApiOutbound/ApiCall audit row via .
/// The audit emission is wrapped in a try/catch that swallows every exception
/// — the audit pipeline is best-effort and must NEVER abort the script's
/// outbound call (alog.md §7). The original
/// (or the original thrown exception) flows back to the caller unchanged.
///
///
/// Helper for external system API calls from scripts.
///
public class ExternalSystemHelper
{
private static readonly Regex HttpStatusRegex = new(
@"HTTP\s+(?\d{3})",
RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly IExternalSystemClient? _client;
private readonly string _instanceName;
private readonly ILogger _logger;
private readonly Guid _executionId;
///
/// (ParentExecutionId): the spawning execution's id when
/// this run was inbound-API-routed; null for non-routed runs.
/// Threaded alongside for audit-row stamping.
///
private readonly Guid? _parentExecutionId;
private readonly IAuditWriter? _auditWriter;
private readonly string _siteId;
private readonly string? _sourceScript;
private readonly ICachedCallTelemetryForwarder? _cachedForwarder;
///
/// SourceNode-stamping: the local cluster node name on
/// which this script is executing (node-a/node-b).
/// Stamped onto SiteCallOperational.SourceNode on the three
/// cached-call telemetry construction sites (CachedSubmit + the two
/// immediate-completion rows) so central can persist it on the
/// SiteCalls row.
///
private readonly string? _sourceNode;
///
/// Flow Debugger (plan 2026-09-06 D5): the run's inline collector. Each emit
/// site hands it the SAME it gives the writer /
/// forwarder, collector first. Null when untraced — the production default.
///
private readonly Action? _traceSink;
///
/// Step logs (design 2026-09-09 D6): whether the owning run is traced. The authority for
/// opening a around . Deliberately NOT
/// : the sink is null whenever the run has no inline collector,
/// which is a different question from "is this run traced" — a traced run without a
/// collector still writes durable ScriptLog rows that must name their step.
///
private readonly bool _isTraced;
///
/// Initializes a new external system helper for script API calls.
///
/// Optional client for external system API calls.
/// Unique name of the instance making the call.
/// Logger for diagnostics and warnings.
/// Unique identifier for this script execution.
/// Optional writer for audit log entries.
/// Identifier of the site where this call originates.
/// Optional name of the source script for audit trail.
/// Optional forwarder for cached call telemetry.
/// Optional identifier of the parent execution (for routed calls).
/// Optional cluster node identifier (node-a/node-b) for audit stamping.
/// Flow Debugger inline collector — receives the same each emit hands to the writer/forwarder; null when untraced.
/// Step logs (design D6): true when the owning run is traced, so pre-mints its row id and publishes it as the ambient step. Defaults to false for direct (test) construction.
// Internal constructor for tests living in ZB.MOM.WW.ScadaBridge.SiteRuntime.Tests
// (via InternalsVisibleTo). Production sites resolve the helper through
// ScriptRuntimeContext.ExternalSystem.
//
// Parameter ordering: executionId sits immediately after the
// ILogger across all four audit-threaded ctors (ExternalSystemHelper,
// DatabaseHelper, AuditingDbConnection, AuditingDbCommand) — a required
// Guid cannot follow the optional provenance params without a
// required-after-optional compile error, so the post-logger slot is the
// one consistent position that compiles cleanly everywhere. The nullable
// parentExecutionId is a trailing optional param so existing positional
// callers stay source-compatible.
internal ExternalSystemHelper(
IExternalSystemClient? client,
string instanceName,
ILogger logger,
Guid executionId,
IAuditWriter? auditWriter = null,
string siteId = "",
string? sourceScript = null,
ICachedCallTelemetryForwarder? cachedForwarder = null,
Guid? parentExecutionId = null,
string? sourceNode = null,
Action? traceSink = null,
bool isTraced = false)
{
_client = client;
_instanceName = instanceName;
_logger = logger;
_executionId = executionId;
_auditWriter = auditWriter;
_siteId = siteId;
_sourceScript = sourceScript;
_cachedForwarder = cachedForwarder;
_parentExecutionId = parentExecutionId;
_sourceNode = sourceNode;
_traceSink = GuardTraceSink(traceSink, logger);
_isTraced = isTraced;
}
///
/// Makes a synchronous external system API call with immediate response.
///
/// Name of the external system to call.
/// Name of the method to invoke on the external system.
/// Optional parameters to pass to the method.
/// Cancellation token for the async operation.
/// Result of the external API call including status and response data.
public async Task Call(
string systemName,
string methodName,
IReadOnlyDictionary? parameters = null,
CancellationToken cancellationToken = default)
{
if (_client == null)
throw new InvalidOperationException("External system client not available");
// Wrap the outbound call so every
// attempt emits exactly one ApiOutbound/ApiCall row. The wrapper
// mirrors the existing call-site behaviour — the original result
// OR original exception flows back to the script untouched; the
// audit emission is best-effort.
var occurredAtUtc = DateTime.UtcNow;
var startTicks = Stopwatch.GetTimestamp();
// Step logs (design D6): pre-mint this call's audit row id and make it the ambient step
// so the helper's own warnings — and anything the client logs through this context —
// are attributed to the API-call node. Gated on the run's trace flag, NOT on the
// presence of an inline collector: a traced run without one still writes durable
// ScriptLog rows that need the step id. Untraced mints nothing and opens no scope.
Guid? stepId = _isTraced ? Guid.NewGuid() : null;
using var step = stepId is { } callStepId ? TraceStepScope.Enter(callStepId) : null;
ExternalCallResult? result = null;
Exception? thrown = null;
try
{
result = await _client.CallAsync(systemName, methodName, parameters, cancellationToken);
return result;
}
catch (Exception ex)
{
thrown = ex;
throw;
}
finally
{
var elapsedMs = (int)((Stopwatch.GetTimestamp() - startTicks)
* 1000d / Stopwatch.Frequency);
EmitCallAudit(systemName, methodName, occurredAtUtc, elapsedMs, result, thrown, parameters, stepId);
}
}
///
/// Submit a cached outbound API call. Mints a
/// fresh , emits the lifecycle's first
/// CachedSubmit telemetry packet, hands the call to the
/// store-and-forward retry loop (which emits per-attempt and terminal
/// telemetry under the same id), and returns
/// the id immediately so the script can later query
/// Tracking.Status(id).
///
///
/// Best-effort emission (alog.md §7): if the forwarder throws,
/// the failure is logged and swallowed; the underlying cached-call
/// path still runs and the id is still returned. The script must never
/// be aborted by an audit-pipeline failure.
///
/// Name of the external system to call.
/// Name of the method to invoke on the external system.
/// Optional parameters to pass to the method.
/// Cancellation token for the async operation.
/// Tracked operation identifier for status queries.
public async Task CachedCall(
string systemName,
string methodName,
IReadOnlyDictionary? parameters = null,
CancellationToken cancellationToken = default)
{
if (_client == null)
throw new InvalidOperationException("External system client not available");
var trackedId = TrackedOperationId.New();
var occurredAtUtc = DateTime.UtcNow;
var target = $"{systemName}.{methodName}";
// Emit CachedSubmit telemetry BEFORE handing off to the S&F
// engine — that way the SiteCalls row is materialised before the
// first delivery attempt and Tracking.Status(id) can observe a
// Submitted row even if the immediate-delivery attempt happens to
// resolve before this method returns.
await EmitCachedSubmitTelemetryAsync(
systemName, methodName, target, trackedId, occurredAtUtc, parameters, cancellationToken)
.ConfigureAwait(false);
// Hand off to the existing cached-call path. The TrackedOperationId
// becomes the S&F message id so the retry loop can read it back
// via StoreAndForwardMessage.Id.
//
// The result is now retained because the
// immediate-success path (WasBuffered=false) bypasses S&F entirely
// — no retry loop, no ICachedCallLifecycleObserver fire. The
// helper must emit the Attempted + CachedResolve terminal rows
// itself, otherwise Tracking.Status(id) would stay in Submitted
// forever and the audit log would be missing the lifecycle.
// The WasBuffered=true path is unaffected — the S&F retry loop
// owns the Attempted + Resolve emissions in that case.
ExternalCallResult? result;
try
{
result = await _client.CachedCallAsync(
systemName,
methodName,
parameters,
_instanceName,
cancellationToken,
trackedId,
// (ExecutionId): thread the script
// execution's ExecutionId + SourceScript so a buffered
// cached call's retry-loop audit rows carry them.
executionId: _executionId,
sourceScript: _sourceScript,
// (ParentExecutionId): thread the
// spawning inbound-API request's ExecutionId so a buffered
// cached call's retry-loop audit rows carry it too. Null
// for a non-routed run.
parentExecutionId: _parentExecutionId).ConfigureAwait(false);
}
catch (Exception ex)
{
// The cached-call surface returns ExternalCallResult on permanent
// failure rather than throwing; a throw here is exceptional
// (e.g. cancellation, resolver outage). Log it and rethrow — the
// script does need to learn about catastrophic failures. The
// tracked id was still returned via the telemetry submit above.
_logger.LogWarning(ex,
"ExternalSystem.CachedCall threw for {System}.{Method} (TrackedOperationId {Id})",
systemName, methodName, trackedId);
throw;
}
// Immediate-completion lifecycle — emit the
// missing Attempted + CachedResolve rows when the underlying call
// resolved without engaging the store-and-forward retry loop.
if (result is { WasBuffered: false })
{
await EmitImmediateTerminalTelemetryAsync(
systemName, methodName, target, trackedId, result, parameters, cancellationToken)
.ConfigureAwait(false);
}
return trackedId;
}
///
/// Best-effort emission of the CachedSubmit lifecycle event. Any
/// exception thrown by the forwarder is logged and swallowed so the
/// calling script's enqueue is not disturbed.
///
private async Task EmitCachedSubmitTelemetryAsync(
string systemName,
string methodName,
string target,
TrackedOperationId trackedId,
DateTime occurredAtUtc,
IReadOnlyDictionary? parameters,
CancellationToken cancellationToken)
{
if (_cachedForwarder == null)
{
return;
}
CachedCallTelemetry telemetry;
try
{
telemetry = new CachedCallTelemetry(
Audit: ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.CachedSubmit,
status: AuditStatus.Submitted,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
target: target,
// CorrelationId stays the per-operation lifecycle id
// (TrackedOperationId); ExecutionId carries the
// per-execution id shared across this script run.
correlationId: trackedId.Value,
executionId: _executionId,
// (ParentExecutionId): the spawning
// execution's id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
// Submit precedes the call — request args only, no response yet.
requestSummary: SerializeRequest(parameters)),
Operational: new SiteCallOperational(
TrackedOperationId: trackedId,
Channel: "ApiOutbound",
Target: target,
SourceSite: _siteId,
// SourceNode-stamping: the local node name
// (node-a/node-b) — threaded through INodeIdentityProvider
// at the launching script run; null when no provider was
// wired so central persists SiteCalls.SourceNode as NULL.
SourceNode: _sourceNode,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: occurredAtUtc,
UpdatedAtUtc: occurredAtUtc,
TerminalAtUtc: null));
}
catch (Exception buildEx)
{
_logger.LogWarning(buildEx,
"Failed to build CachedSubmit telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission",
systemName, methodName, trackedId);
return;
}
try
{
// Flow Debugger: collector first, then the forwarder.
_traceSink?.Invoke(telemetry.Audit);
await _cachedForwarder.ForwardAsync(telemetry, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"CachedSubmit telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})",
systemName, methodName, trackedId);
}
}
///
/// Emit the Attempted + CachedResolve lifecycle
/// rows for an immediate-completion CachedCall (WasBuffered=false).
/// The S&F retry loop never engaged, so the
/// ICachedCallLifecycleObserver never fires — the helper must
/// produce both rows itself to keep the audit contract whole
/// (Submit → Attempted → Resolve under one TrackedOperationId).
///
///
/// Best-effort emission: a throwing forwarder is logged and swallowed
/// per alog.md §7. The two rows are emitted INDEPENDENTLY so a single
/// forwarder fault doesn't drop both halves of the terminal pair.
///
private async Task EmitImmediateTerminalTelemetryAsync(
string systemName,
string methodName,
string target,
TrackedOperationId trackedId,
ExternalCallResult result,
IReadOnlyDictionary? parameters,
CancellationToken cancellationToken)
{
if (_cachedForwarder == null)
{
return;
}
var occurredAtUtc = DateTime.UtcNow;
// Extract an HTTP status from the error message when present
// (mirrors EmitCallAudit's existing HttpStatusRegex behaviour so
// the immediate-failure row carries the same HttpStatus value the
// synchronous Call() audit row would have stamped).
int? httpStatus = null;
if (!result.Success && !string.IsNullOrEmpty(result.ErrorMessage))
{
var match = HttpStatusRegex.Match(result.ErrorMessage);
if (match.Success && int.TryParse(match.Groups["code"].Value, out var code))
{
httpStatus = code;
}
}
// Status mapping for immediate completion:
// Success=true -> Delivered (audit) / "Delivered" (operational)
// Success=false -> Failed (audit) / "Failed" (operational)
// Permanent vs transient is not relevant here: a permanent failure
// returns Success=false WasBuffered=false (parked-equivalent); a
// transient failure with NO S&F engine wired likewise lands here
// with Success=false. Either way the terminal state is "the
// immediate attempt failed and the operation is done".
var auditTerminalStatus = result.Success
? AuditStatus.Delivered
: AuditStatus.Failed;
var operationalTerminalStatus = result.Success ? "Delivered" : "Failed";
// --- Attempted row -------------------------------------------------
CachedCallTelemetry attempted;
try
{
attempted = new CachedCallTelemetry(
Audit: ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCallCached,
status: AuditStatus.Attempted,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
target: target,
// CorrelationId = per-operation lifecycle id;
// ExecutionId = per-execution id for this script run.
correlationId: trackedId.Value,
executionId: _executionId,
// (ParentExecutionId): the spawning
// execution's id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
httpStatus: httpStatus,
errorMessage: result.Success ? null : result.ErrorMessage,
requestSummary: SerializeRequest(parameters),
responseSummary: result.ResponseJson),
Operational: new SiteCallOperational(
TrackedOperationId: trackedId,
Channel: "ApiOutbound",
Target: target,
SourceSite: _siteId,
// SourceNode-stamping: the local node name
// (node-a/node-b) — threaded through INodeIdentityProvider
// at the launching script run; null when no provider was
// wired so central persists SiteCalls.SourceNode as NULL.
SourceNode: _sourceNode,
Status: "Attempted",
// RetryCount stays 0 — the operation never reached the
// S&F retry sweep, so no retries were performed.
RetryCount: 0,
LastError: result.Success ? null : result.ErrorMessage,
HttpStatus: httpStatus,
CreatedAtUtc: occurredAtUtc,
UpdatedAtUtc: occurredAtUtc,
TerminalAtUtc: null));
}
catch (Exception buildEx)
{
_logger.LogWarning(buildEx,
"Failed to build immediate-Attempted telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission",
systemName, methodName, trackedId);
attempted = null!;
}
if (attempted is not null)
{
try
{
// Flow Debugger: the immediate-completion rows are emitted inline while
// the run is still open, so they legitimately ride the collector.
_traceSink?.Invoke(attempted.Audit);
await _cachedForwarder.ForwardAsync(attempted, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Immediate-Attempted telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})",
systemName, methodName, trackedId);
}
}
// --- CachedResolve row --------------------------------------------
CachedCallTelemetry resolve;
try
{
resolve = new CachedCallTelemetry(
Audit: ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.CachedResolve,
status: auditTerminalStatus,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
target: target,
// CorrelationId = per-operation lifecycle id;
// ExecutionId = per-execution id for this script run.
correlationId: trackedId.Value,
executionId: _executionId,
// (ParentExecutionId): the spawning
// execution's id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
httpStatus: httpStatus,
errorMessage: result.Success ? null : result.ErrorMessage,
requestSummary: SerializeRequest(parameters),
responseSummary: result.ResponseJson),
Operational: new SiteCallOperational(
TrackedOperationId: trackedId,
Channel: "ApiOutbound",
Target: target,
SourceSite: _siteId,
// SourceNode-stamping: the local node name
// (node-a/node-b) — threaded through INodeIdentityProvider
// at the launching script run; null when no provider was
// wired so central persists SiteCalls.SourceNode as NULL.
SourceNode: _sourceNode,
Status: operationalTerminalStatus,
RetryCount: 0,
LastError: result.Success ? null : result.ErrorMessage,
HttpStatus: httpStatus,
CreatedAtUtc: occurredAtUtc,
UpdatedAtUtc: occurredAtUtc,
// Immediate-completion terminal — mark TerminalAtUtc so
// SiteCallAudit can move the row to its purge eligible
// set.
TerminalAtUtc: occurredAtUtc));
}
catch (Exception buildEx)
{
_logger.LogWarning(buildEx,
"Failed to build immediate-CachedResolve telemetry for {System}.{Method} (TrackedOperationId {Id}) — skipping emission",
systemName, methodName, trackedId);
return;
}
try
{
_traceSink?.Invoke(resolve.Audit);
await _cachedForwarder.ForwardAsync(resolve, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Immediate-CachedResolve telemetry forward failed for {System}.{Method} (TrackedOperationId {Id})",
systemName, methodName, trackedId);
}
}
///
/// Best-effort emission of one ApiOutbound/ApiCall audit
/// row. Any exception thrown by the writer is logged and swallowed —
/// audit-write failures must never abort the user-facing action.
///
/// External system name.
/// Method invoked on it.
/// When the call started.
/// How long it took.
/// The call result, when one was produced.
/// The exception the call threw, if any.
/// The call arguments.
///
/// Step logs (design 2026-09-09 D6): the PRE-MINTED row id also opened as
/// the ambient , so the log lines produced during the call name
/// this row. Null (untraced) mints a fresh id as before.
///
private void EmitCallAudit(
string systemName,
string methodName,
DateTime occurredAtUtc,
int durationMs,
ExternalCallResult? result,
Exception? thrown,
IReadOnlyDictionary? parameters,
Guid? eventId = null)
{
if (_auditWriter == null)
{
return;
}
AuditEvent evt;
try
{
evt = BuildCallAuditEvent(
systemName, methodName, occurredAtUtc, durationMs, result, thrown, parameters, eventId);
}
catch (Exception buildEx)
{
// Building the event itself must never propagate. This is a
// defensive guard — populating a record from already-validated
// values shouldn't throw, but we honour the alog.md §7
// best-effort contract regardless.
_logger.LogWarning(buildEx,
"Failed to build Audit Log #23 event for {System}.{Method} — skipping emission",
systemName, methodName);
return;
}
try
{
// Flow Debugger: collector first (same instance), then the writer.
_traceSink?.Invoke(evt);
// Fire-and-forget so we never block the script on the audit
// writer; the writer itself is responsible for fast, durable
// enqueue (site SQLite hot-path). We DO observe failures via
// ContinueWith so a thrown writer is logged rather than going
// to the unobserved-task firehose.
var writeTask = _auditWriter.WriteAsync(evt, CancellationToken.None);
if (!writeTask.IsCompleted)
{
writeTask.ContinueWith(
t => _logger.LogWarning(t.Exception,
"Audit Log #23 write failed for EventId {EventId} ({System}.{Method})",
evt.EventId, systemName, methodName),
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
else if (writeTask.IsFaulted)
{
_logger.LogWarning(writeTask.Exception,
"Audit Log #23 write failed for EventId {EventId} ({System}.{Method})",
evt.EventId, systemName, methodName);
}
}
catch (Exception writeEx)
{
// Synchronous throw from WriteAsync (e.g. ArgumentNullException
// before the writer's own try/catch). Swallow + log per the
// alog.md §7 contract.
_logger.LogWarning(writeEx,
"Audit Log #23 write threw synchronously for EventId {EventId} ({System}.{Method})",
evt.EventId, systemName, methodName);
}
}
private AuditEvent BuildCallAuditEvent(
string systemName,
string methodName,
DateTime occurredAtUtc,
int durationMs,
ExternalCallResult? result,
Exception? thrown,
IReadOnlyDictionary? parameters,
Guid? eventId = null)
{
// Status: Delivered on a Success result; Failed otherwise (the
// ExternalSystemClient already maps HTTP non-2xx + transient
// exceptions into Success=false on the result, or surfaces a raw
// exception). No distinction is made between transient + permanent
// failure here — both manifest as Status.Failed on the sync path.
var status = (thrown == null && result != null && result.Success)
? AuditStatus.Delivered
: AuditStatus.Failed;
string? errorMessage = null;
string? errorDetail = null;
int? httpStatus = null;
if (thrown != null)
{
errorMessage = thrown.Message;
errorDetail = thrown.ToString();
}
else if (result != null && !result.Success)
{
errorMessage = result.ErrorMessage;
// The ExternalSystemClient embeds the HTTP status code in the
// error message as "HTTP {code}". Parse it back out so the
// audit row carries the structured value.
if (!string.IsNullOrEmpty(result.ErrorMessage))
{
var match = HttpStatusRegex.Match(result.ErrorMessage);
if (match.Success
&& int.TryParse(match.Groups["code"].Value, out var parsed))
{
httpStatus = parsed;
}
}
}
return ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCall,
status: status,
// Step logs (D6): the id the caller already published as the ambient step.
eventId: eventId,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
// Outbound channel: per the Audit Log Actor-column spec the actor
// is the calling script. Null when no single script owns the call
// (e.g. a shared script running inline).
actor: _sourceScript,
target: $"{systemName}.{methodName}",
// A sync one-shot call has no operation
// lifecycle, so CorrelationId is null. ExecutionId carries the
// per-execution id so all the sync ApiCall/DbWrite rows from
// one script run can be correlated together.
correlationId: null,
executionId: _executionId,
// (ParentExecutionId): the spawning execution's
// id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
httpStatus: httpStatus,
durationMs: durationMs,
errorMessage: errorMessage,
errorDetail: errorDetail,
// Payload capture: the request arguments and the response body.
// The audit writer's redactor applies the configured size cap and
// header/secret redaction downstream — the emitter just hands
// over the raw values.
requestSummary: SerializeRequest(parameters),
responseSummary: result?.ResponseJson,
payloadTruncated: false,
extra: null);
}
///
/// Serialises the outbound-call argument dictionary into the JSON
/// RequestSummary stamped on ApiOutbound audit rows.
/// Returns null for a null/empty argument set. Serialization
/// failure is swallowed (returns null) — a payload that cannot be
/// summarised must never abort the best-effort audit emission.
///
private static string? SerializeRequest(IReadOnlyDictionary? parameters)
{
if (parameters is null || parameters.Count == 0)
{
return null;
}
try
{
return JsonSerializer.Serialize(parameters);
}
catch (Exception)
{
return null;
}
}
}
///
/// Helper for Database.Connection/CachedWrite syntax.
///
///
/// mirrors
/// — mints a
/// , emits the lifecycle's first
/// CachedSubmit packet (Channel DbOutbound), hands off to the S&F
/// retry loop, and returns the id. Per-attempt + terminal telemetry is
/// emitted by the retry loop (Tasks E4/E5).
///
public class DatabaseHelper
{
private readonly IDatabaseGateway? _gateway;
private readonly string _instanceName;
private readonly ILogger _logger;
private readonly Guid _executionId;
///
/// (ParentExecutionId): the spawning execution's id when
/// this run was inbound-API-routed; null for non-routed runs.
/// Threaded alongside for audit-row stamping.
///
private readonly Guid? _parentExecutionId;
private readonly string _siteId;
private readonly string? _sourceScript;
private readonly ICachedCallTelemetryForwarder? _cachedForwarder;
///
/// Best-effort emitter for synchronous
/// Database.Connection-routed Execute / ExecuteScalar /
/// ExecuteReader calls. When wired, returns
/// an that intercepts each command
/// execution and writes one DbOutbound/DbWrite audit
/// row. Optional — when null the helper falls back to the raw
/// inner the gateway
/// returns (tests / minimal hosts that don't wire audit).
///
private readonly IAuditWriter? _auditWriter;
///
/// SourceNode-stamping: the local cluster node name on
/// which this script is executing (node-a/node-b).
/// Stamped onto SiteCallOperational.SourceNode at the
/// Database.CachedWrite CachedSubmit telemetry construction
/// site so central can persist it on the SiteCalls row.
///
private readonly string? _sourceNode;
///
/// Flow Debugger (plan 2026-09-06 D5): the run's inline collector, handed the
/// same each emit gives the writer / forwarder and
/// threaded into for the sync DbWrite
/// rows. Null when untraced.
///
private readonly Action? _traceSink;
///
/// Initializes a new database helper for script database access.
///
/// Optional gateway for database connection and cached write access.
/// Unique name of the instance making the call.
/// Logger for diagnostics and warnings.
/// Unique identifier for this script execution.
/// Optional writer for audit log entries.
/// Identifier of the site where this call originates.
/// Optional name of the source script for audit trail.
/// Optional forwarder for cached call telemetry.
/// Optional identifier of the parent execution (for routed calls).
/// Optional cluster node identifier (node-a/node-b) for audit stamping.
/// Flow Debugger inline collector — receives the same each emit hands to the writer/forwarder; null when untraced.
// Parameter ordering: executionId sits immediately after the
// ILogger — see the note on ExternalSystemHelper's ctor for why the
// post-logger slot is the one consistent position across all four
// audit-threaded ctors. parentExecutionId is a trailing optional param.
internal DatabaseHelper(
IDatabaseGateway? gateway,
string instanceName,
ILogger logger,
Guid executionId,
IAuditWriter? auditWriter = null,
string siteId = "",
string? sourceScript = null,
ICachedCallTelemetryForwarder? cachedForwarder = null,
Guid? parentExecutionId = null,
string? sourceNode = null,
Action? traceSink = null)
{
_gateway = gateway;
_instanceName = instanceName;
_logger = logger;
_executionId = executionId;
_auditWriter = auditWriter;
_siteId = siteId;
_sourceScript = sourceScript;
_cachedForwarder = cachedForwarder;
_parentExecutionId = parentExecutionId;
_sourceNode = sourceNode;
_traceSink = GuardTraceSink(traceSink, logger);
}
///
/// Gets a database connection by name, optionally wrapped with audit logging.
///
/// Name of the database connection to retrieve.
/// Cancellation token for the async operation.
/// An auditing-wrapped database connection ready for use.
public async Task Connection(
string name,
CancellationToken cancellationToken = default)
{
if (_gateway == null)
throw new InvalidOperationException("Database gateway not available");
var inner = await _gateway.GetConnectionAsync(name, cancellationToken);
// Wrap in an auditing decorator so
// every script-initiated Execute* / ExecuteReader on the returned
// connection emits one DbOutbound/DbWrite audit row. The wrapper
// delegates all other ADO.NET behaviour to the inner connection
// unchanged — including disposal, so the caller's existing
// dispose pattern (await using var conn = ...) still releases
// the underlying connection to the pool.
if (_auditWriter == null)
{
return inner;
}
return new AuditingDbConnection(
inner,
_auditWriter,
connectionName: name,
siteId: _siteId,
instanceName: _instanceName,
sourceScript: _sourceScript,
logger: _logger,
executionId: _executionId,
// (ParentExecutionId): the spawning execution's
// id, threaded alongside _executionId. Null for non-routed runs.
parentExecutionId: _parentExecutionId,
// Flow Debugger: the sync DbWrite rows ride the inline collector.
onAudit: _traceSink);
}
///
/// Submit a cached outbound database write. Mints a fresh
/// , emits CachedSubmit telemetry on
/// DbOutbound, hands off to the cached-write S&F path, and
/// returns the id. Best-effort emission per alog.md §7.
///
/// Name of the database connection to use.
/// SQL statement to execute.
/// Optional parameters to pass to the SQL statement.
/// Cancellation token for the async operation.
/// Tracked operation identifier for status queries.
public async Task CachedWrite(
string name,
string sql,
IReadOnlyDictionary? parameters = null,
CancellationToken cancellationToken = default)
{
if (_gateway == null)
throw new InvalidOperationException("Database gateway not available");
var trackedId = TrackedOperationId.New();
var occurredAtUtc = DateTime.UtcNow;
// The DB cached-write target uses the connection name (the only
// human-readable handle the gateway carries on the buffered row).
var target = name;
await EmitCachedDbSubmitTelemetryAsync(
name, trackedId, target, occurredAtUtc, cancellationToken)
.ConfigureAwait(false);
// The gateway now attempts the write immediately and
// classifies the outcome (mirroring ExternalSystem.CachedCall). The
// result is retained because the immediate paths (WasBuffered=false —
// immediate success OR a synchronous permanent failure) bypass the
// S&F retry loop entirely, so no retry-loop telemetry ever fires.
// This helper must emit the Attempted + CachedResolve terminal rows
// itself, otherwise Tracking.Status(id) would stay Submitted forever
// and the audit log would be missing the terminal lifecycle. The
// WasBuffered=true path is unaffected — the S&F retry loop owns the
// Attempted + Resolve emissions there.
ExternalCallResult? result;
try
{
result = await _gateway.CachedWriteAsync(
name, sql, parameters, _instanceName, cancellationToken, trackedId,
// (ExecutionId): thread the script
// execution's ExecutionId + SourceScript so a buffered
// cached write's retry-loop audit rows carry them.
executionId: _executionId,
sourceScript: _sourceScript,
// (ParentExecutionId): thread the
// spawning inbound-API request's ExecutionId so a buffered
// cached write's retry-loop audit rows carry it too. Null
// for a non-routed run.
parentExecutionId: _parentExecutionId)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Database.CachedWrite threw for {Connection} (TrackedOperationId {Id})",
name, trackedId);
throw;
}
// Immediate-completion lifecycle — emit the missing
// Attempted + CachedResolve rows when the underlying write resolved
// without engaging the store-and-forward retry loop (immediate
// success or a synchronous permanent failure).
if (result is { WasBuffered: false })
{
await EmitImmediateDbTerminalTelemetryAsync(
name, target, trackedId, result, cancellationToken)
.ConfigureAwait(false);
}
return trackedId;
}
///
/// Best-effort emission of the immediate-completion lifecycle
/// for a Database.CachedWrite that resolved without the S&F
/// retry loop — emits an Attempted row then a terminal
/// CachedResolve row (Delivered on success, Failed on
/// a synchronous permanent SQL error). The DB parallel of
/// . Any forwarder
/// failure is logged and swallowed (alog.md §7).
///
private async Task EmitImmediateDbTerminalTelemetryAsync(
string connectionName,
string target,
TrackedOperationId trackedId,
ExternalCallResult result,
CancellationToken cancellationToken)
{
if (_cachedForwarder == null)
{
return;
}
var occurredAtUtc = DateTime.UtcNow;
// Status mapping mirrors the API path: success -> Delivered, a
// synchronous permanent failure -> Failed. A transient failure never
// reaches here (WasBuffered=true), so "the immediate attempt failed
// and the operation is done" always means a permanent failure.
var auditTerminalStatus = result.Success ? AuditStatus.Delivered : AuditStatus.Failed;
var operationalTerminalStatus = result.Success ? "Delivered" : "Failed";
// --- Attempted row -------------------------------------------------
CachedCallTelemetry? attempted = TryBuildDbTerminalTelemetry(
connectionName, target, trackedId, occurredAtUtc,
AuditKind.DbWriteCached, AuditStatus.Attempted, "Attempted",
result, isTerminal: false);
if (attempted is not null)
{
try
{
// Flow Debugger: emitted inline while the run is open — rides the collector.
_traceSink?.Invoke(attempted.Audit);
await _cachedForwarder.ForwardAsync(attempted, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Immediate-Attempted telemetry forward failed for Database.CachedWrite {Connection} (TrackedOperationId {Id})",
connectionName, trackedId);
}
}
// --- CachedResolve row --------------------------------------------
CachedCallTelemetry? resolve = TryBuildDbTerminalTelemetry(
connectionName, target, trackedId, occurredAtUtc,
AuditKind.CachedResolve, auditTerminalStatus, operationalTerminalStatus,
result, isTerminal: true);
if (resolve is not null)
{
try
{
_traceSink?.Invoke(resolve.Audit);
await _cachedForwarder.ForwardAsync(resolve, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"Immediate-CachedResolve telemetry forward failed for Database.CachedWrite {Connection} (TrackedOperationId {Id})",
connectionName, trackedId);
}
}
}
///
/// Builds one immediate-completion DbOutbound telemetry packet, or
/// returns null (and logs) when construction throws — so a build
/// failure skips emission rather than aborting the script.
///
private CachedCallTelemetry? TryBuildDbTerminalTelemetry(
string connectionName,
string target,
TrackedOperationId trackedId,
DateTime occurredAtUtc,
AuditKind kind,
AuditStatus auditStatus,
string operationalStatus,
ExternalCallResult result,
bool isTerminal)
{
try
{
return new CachedCallTelemetry(
Audit: ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.DbOutbound,
kind: kind,
status: auditStatus,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
target: target,
correlationId: trackedId.Value,
executionId: _executionId,
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
errorMessage: result.Success ? null : result.ErrorMessage),
Operational: new SiteCallOperational(
TrackedOperationId: trackedId,
Channel: "DbOutbound",
Target: target,
SourceSite: _siteId,
SourceNode: _sourceNode,
Status: operationalStatus,
RetryCount: 0,
LastError: result.Success ? null : result.ErrorMessage,
HttpStatus: null,
CreatedAtUtc: occurredAtUtc,
UpdatedAtUtc: occurredAtUtc,
TerminalAtUtc: isTerminal ? occurredAtUtc : null));
}
catch (Exception buildEx)
{
_logger.LogWarning(buildEx,
"Failed to build immediate-{Kind} telemetry for Database.CachedWrite {Connection} (TrackedOperationId {Id}) — skipping emission",
kind, connectionName, trackedId);
return null;
}
}
private async Task EmitCachedDbSubmitTelemetryAsync(
string connectionName,
TrackedOperationId trackedId,
string target,
DateTime occurredAtUtc,
CancellationToken cancellationToken)
{
if (_cachedForwarder == null)
{
return;
}
CachedCallTelemetry telemetry;
try
{
telemetry = new CachedCallTelemetry(
Audit: ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.DbOutbound,
kind: AuditKind.CachedSubmit,
status: AuditStatus.Submitted,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
target: target,
// CorrelationId = per-operation lifecycle id
// (TrackedOperationId); ExecutionId = per-execution id.
correlationId: trackedId.Value,
executionId: _executionId,
// (ParentExecutionId): the spawning
// execution's id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript),
Operational: new SiteCallOperational(
TrackedOperationId: trackedId,
Channel: "DbOutbound",
Target: target,
SourceSite: _siteId,
// SourceNode-stamping: the local node name
// (node-a/node-b) — threaded through INodeIdentityProvider
// at the launching script run; null when no provider was
// wired so central persists SiteCalls.SourceNode as NULL.
SourceNode: _sourceNode,
Status: "Submitted",
RetryCount: 0,
LastError: null,
HttpStatus: null,
CreatedAtUtc: occurredAtUtc,
UpdatedAtUtc: occurredAtUtc,
TerminalAtUtc: null));
}
catch (Exception buildEx)
{
_logger.LogWarning(buildEx,
"Failed to build CachedSubmit telemetry for Database.CachedWrite {Connection} (TrackedOperationId {Id}) — skipping emission",
connectionName, trackedId);
return;
}
try
{
// Flow Debugger: collector first, then the forwarder.
_traceSink?.Invoke(telemetry.Audit);
await _cachedForwarder.ForwardAsync(telemetry, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex,
"CachedSubmit telemetry forward failed for Database.CachedWrite {Connection} (TrackedOperationId {Id})",
connectionName, trackedId);
}
}
}
///
/// Notification Outbox: helper for the Notify script API.
///
/// In the outbox design the site no longer delivers notification email inline.
/// Notify.To("listName").Send(...) enqueues the notification into the site
/// Store-and-Forward Engine — which forwards it to central — and returns a
/// NotificationId handle immediately. Notify.Status(id) later queries
/// the delivery status of that notification.
///
public class NotifyHelper
{
private readonly StoreAndForwardService? _storeAndForward;
private readonly ICanTell? _siteCommunicationActor;
private readonly string _siteId;
private readonly string _instanceName;
private readonly string? _sourceScript;
private readonly TimeSpan _askTimeout;
private readonly ILogger _logger;
///
/// The per-execution id for this script run, stamped
/// into AuditEvent.ExecutionId on the NotifySend row.
///
private readonly Guid _executionId;
///
/// (ParentExecutionId): the spawning execution's id when
/// this run was inbound-API-routed; null for non-routed runs.
/// Threaded alongside for audit-row stamping.
///
private readonly Guid? _parentExecutionId;
///
/// Best-effort emitter for the
/// Notification/NotifySend row produced when the script
/// calls Notify.To(list).Send(...). Optional — when null the
/// degrades to a no-op audit path so tests
/// / minimal hosts that don't wire AddAuditLog still work (mirrors the
/// IExternalSystemClient wrapper).
///
private readonly IAuditWriter? _auditWriter;
///
/// SourceNode-stamping: the cluster node name on which this
/// script is executing — node-a/node-b. Stamped onto
/// NotificationSubmit.SourceNode by
/// so central can persist it on the Notifications row.
///
private readonly string? _sourceNode;
///
/// Flow Debugger (plan 2026-09-06 D5): the run's inline collector, forwarded to
/// each . Null when untraced.
///
private readonly Action? _traceSink;
///
/// Initializes a new notification helper for script notification delivery.
///
/// Optional store-and-forward service for notification delivery.
/// Optional actor for site-to-central communication.
/// Identifier of the site where this notification originates.
/// Unique name of the instance sending the notification.
/// Optional name of the source script for audit trail.
/// Timeout for Ask pattern messages.
/// Logger for diagnostics and warnings.
/// Unique identifier for this script execution.
/// Optional writer for audit log entries.
/// Optional identifier of the parent execution (for routed calls).
/// Optional cluster node identifier (node-a/node-b) for audit stamping.
/// Flow Debugger inline collector — receives the same Send hands to the writer; null when untraced.
// Parameter ordering: executionId sits immediately after the ILogger,
// consistent with the other audit-threaded ctors. parentExecutionId is
// a trailing optional param.
internal NotifyHelper(
StoreAndForwardService? storeAndForward,
ICanTell? siteCommunicationActor,
string siteId,
string instanceName,
string? sourceScript,
TimeSpan askTimeout,
ILogger logger,
Guid executionId,
IAuditWriter? auditWriter = null,
Guid? parentExecutionId = null,
string? sourceNode = null,
Action? traceSink = null)
{
_storeAndForward = storeAndForward;
_siteCommunicationActor = siteCommunicationActor;
_siteId = siteId;
_instanceName = instanceName;
_sourceScript = sourceScript;
_askTimeout = askTimeout;
_logger = logger;
_executionId = executionId;
_auditWriter = auditWriter;
_parentExecutionId = parentExecutionId;
_sourceNode = sourceNode;
_traceSink = GuardTraceSink(traceSink, logger);
}
///
/// Selects the notification list to send to.
///
/// Name of the notification list to target.
/// A notification target ready for message composition and sending.
public NotifyTarget To(string listName)
{
return new NotifyTarget(
listName, _storeAndForward, _siteId, _instanceName, _sourceScript, _logger,
// The per-execution id stamped into the
// NotifySend row's ExecutionId column.
_executionId,
// Forward the writer so Send()
// can emit one NotifySend(Submitted) row per accepted submission.
_auditWriter,
// (ParentExecutionId): the spawning execution's
// id, threaded alongside _executionId. Null for non-routed runs.
_parentExecutionId,
// SourceNode-stamping: the local node name, stamped
// onto NotificationSubmit.SourceNode in Send().
_sourceNode,
// Flow Debugger: the NotifySend row rides the inline collector.
traceSink: _traceSink);
}
///
/// Queries the delivery status of a previously-sent notification.
///
/// The query is issued to central via the site communication actor. While the
/// notification is still buffered in the site Store-and-Forward Engine — central
/// has no row for it yet (Found: false) but the buffer still holds the id —
/// the status is reported as the site-local Forwarding state. If central
/// has a row, its status is mapped through verbatim. If central does not know the
/// id and it is not buffered locally, the status is Unknown.
///
/// The notification identifier returned from Send.
/// The delivery status of the notification.
public async Task Status(string notificationId)
{
if (_siteCommunicationActor == null)
throw new InvalidOperationException(
"Notification status query is not available — site communication actor not wired");
var correlationId = Guid.NewGuid().ToString();
var query = new NotificationStatusQuery(correlationId, notificationId);
NotificationStatusResponse response;
try
{
response = await _siteCommunicationActor
.Ask(query, _askTimeout);
}
catch (Exception ex)
{
// Central could not be reached. Fall through to the buffer check: if the
// notification is still in the local S&F buffer it is Forwarding.
_logger.LogWarning(ex,
"Notification status query for {NotificationId} did not reach central",
notificationId);
response = new NotificationStatusResponse(
correlationId, Found: false, Status: "Unknown",
RetryCount: 0, LastError: null, DeliveredAt: null);
}
if (response.Found)
{
return new NotificationDeliveryStatus(
response.Status, response.RetryCount, response.LastError, response.DeliveredAt);
}
// Central has no row. If the notification is still buffered at the site it
// is in transit — report the site-local Forwarding state. Otherwise it is
// genuinely unknown (never sent, or already forwarded and central lost it).
if (_storeAndForward != null)
{
var buffered = await _storeAndForward.GetMessageByIdAsync(notificationId);
if (buffered != null)
{
return new NotificationDeliveryStatus(
"Forwarding", buffered.RetryCount, buffered.LastError, DeliveredAt: null);
}
}
return new NotificationDeliveryStatus("Unknown", 0, null, null);
}
}
///
/// Notification Outbox: target for Notify.To("listName").Send(...).
///
public class NotifyTarget
{
private readonly string _listName;
private readonly StoreAndForwardService? _storeAndForward;
private readonly string _siteId;
private readonly string _instanceName;
private readonly string? _sourceScript;
private readonly ILogger _logger;
///
/// The per-execution id for this script run, stamped
/// into AuditEvent.ExecutionId on the NotifySend row.
///
private readonly Guid _executionId;
///
/// (ParentExecutionId): the spawning execution's id when
/// this run was inbound-API-routed; null for non-routed runs.
/// Threaded alongside for audit-row stamping.
///
private readonly Guid? _parentExecutionId;
///
/// Best-effort emitter for the
/// Notification/NotifySend row written immediately after
/// the underlying S&F enqueue accepts the submission. Optional —
/// when null no audit row is emitted (no-op path).
///
private readonly IAuditWriter? _auditWriter;
///
/// SourceNode-stamping: the cluster node name on which this
/// script is executing (node-a/node-b). Stamped onto the
/// NotificationSubmit.SourceNode field in so
/// the central NotificationOutboxActor can persist it on the
/// Notifications row.
///
private readonly string? _sourceNode;
///
/// Flow Debugger (plan 2026-09-06 D5): the run's inline collector — handed the
/// same NotifySend the writer receives. Null when
/// untraced.
///
private readonly Action? _traceSink;
///
/// Initializes a new notification target for a specific notification list.
///
/// Name of the notification list to target.
/// Optional store-and-forward service for notification delivery.
/// Identifier of the site where this notification originates.
/// Unique name of the instance sending the notification.
/// Optional name of the source script for audit trail.
/// Logger for diagnostics and warnings.
/// Unique identifier for this script execution.
/// Optional writer for audit log entries.
/// Optional identifier of the parent execution (for routed calls).
/// Optional cluster node identifier (node-a/node-b) for audit stamping.
/// Flow Debugger inline collector; null when untraced.
internal NotifyTarget(
string listName,
StoreAndForwardService? storeAndForward,
string siteId,
string instanceName,
string? sourceScript,
ILogger logger,
Guid executionId,
IAuditWriter? auditWriter = null,
Guid? parentExecutionId = null,
string? sourceNode = null,
Action? traceSink = null)
{
_listName = listName;
_storeAndForward = storeAndForward;
_siteId = siteId;
_instanceName = instanceName;
_sourceScript = sourceScript;
_logger = logger;
_executionId = executionId;
_auditWriter = auditWriter;
_parentExecutionId = parentExecutionId;
_sourceNode = sourceNode;
_traceSink = GuardTraceSink(traceSink, logger);
}
///
/// Enqueues a notification for central delivery and returns its
/// NotificationId immediately.
///
/// The notification is buffered into the site Store-and-Forward Engine under the
/// category; the S&F
/// engine's NotificationForwarder forwards it to central and treats
/// central's ack as the delivery outcome. The returned NotificationId is
/// the single idempotency key end-to-end: it is the S&F message id, it is
/// carried inside the buffered payload, and it is the id the forwarder submits to
/// central. Pass it to to track delivery.
///
/// Subject line for the notification.
/// Message body for the notification.
/// Cancellation token for the async operation.
/// The notification identifier for status tracking.
public async Task Send(
string subject,
string message,
CancellationToken cancellationToken = default)
{
if (_storeAndForward == null)
throw new InvalidOperationException(
"Notification store-and-forward engine not available");
// The script controls the idempotency key: generate the NotificationId here,
// use it as the S&F message id, and carry it inside the buffered payload so
// the forwarder submits the same id to central on every retry.
var notificationId = Guid.NewGuid().ToString("N");
var payload = new NotificationSubmit(
NotificationId: notificationId,
ListName: _listName,
Subject: subject,
Body: message,
// SourceSiteId is re-stamped by the forwarder from its own site id; this
// value is the best-effort site id known to the script runtime.
SourceSiteId: _siteId,
SourceInstanceId: _instanceName,
// SourceScript (FU3): identifier of the script that raised this
// notification, threaded down from the script-execution context for the
// central audit trail. Null when no single script owns the context.
SourceScript: _sourceScript,
SiteEnqueuedAt: DateTimeOffset.UtcNow,
// OriginExecutionId: the SAME per-execution id stamped
// onto this run's NotifySend audit row. It rides inside the serialized
// payload through the S&F buffer to central, where the dispatcher echoes
// it onto the NotifyDeliver rows so all rows for one run share an id.
OriginExecutionId: _executionId,
// OriginParentExecutionId: the SAME parent-execution id
// stamped onto this run's NotifySend audit row — the spawning run's id
// for an inbound-API-routed execution, null otherwise. It rides through
// the S&F buffer to central, where the dispatcher echoes it onto the
// NotifyDeliver rows so the central rows carry the routed run's parent id.
OriginParentExecutionId: _parentExecutionId,
// SourceNode-stamping: the cluster node name on which this
// notification was emitted (node-a/node-b). Stamped from the local
// INodeIdentityProvider via the launching script run. Rides inside the
// serialized payload through the S&F buffer to central, where
// NotificationOutboxActor persists it on the Notifications row.
SourceNode: _sourceNode);
var payloadJson = JsonSerializer.Serialize(payload);
// The S&F engine assigns its own GUID to the message; pin the message id to
// the NotificationId so the buffer can be queried by it (Notify.Status) and
// the forwarder's idempotency key matches the buffered row.
var occurredAtUtc = DateTime.UtcNow;
await _storeAndForward.EnqueueAsync(
StoreAndForwardCategory.Notification,
target: _listName,
payloadJson: payloadJson,
originInstanceName: _instanceName,
// 0 = the documented "no limit" escape hatch: notifications are
// retried until central acks and are never parked for retry
// exhaustion — a long central outage must not strand them behind
// per-message operator unparking. (Corrupt payloads still park.)
maxRetries: 0,
// Never run the forwarder's 30s central Ask inline on the script
// thread: buffer due-immediately and kick the sweep. Send returns
// in milliseconds whether central is up or down.
deferToSweep: true,
messageId: notificationId);
_logger.LogDebug(
"Notify enqueued notification {NotificationId} to list '{List}' for central delivery",
notificationId, _listName);
// Emit one Notification/NotifySend
// (Submitted) row per accepted submission. The emission is wired
// AFTER the EnqueueAsync returns so we only audit submissions the
// S&F engine accepted — a failed enqueue throws, never produces an
// audit row (mirrors ESG: audit fires after the boundary call
// returned a result, never speculatively). Best-effort per alog.md
// §7 — the audit write is wrapped in try/catch and any failure is
// logged + swallowed so the script's Send call still returns the
// NotificationId.
EmitNotifySendAudit(notificationId, subject, message, occurredAtUtc);
return notificationId;
}
///
/// Best-effort emission of one Notification/NotifySend
/// (Status Submitted) audit row. Any exception thrown by the
/// writer is logged and swallowed — audit-write failures must never
/// abort the user-facing Notify.Send call (alog.md §7).
///
private void EmitNotifySendAudit(
string notificationId,
string subject,
string body,
DateTime occurredAtUtc)
{
if (_auditWriter == null)
{
return;
}
AuditEvent evt;
try
{
// CorrelationId is the NotificationId parsed as a Guid. Notify
// mints the id via Guid.NewGuid().ToString("N") so the parse
// is expected to succeed; on the off-chance the format
// changes / a caller injects an unparseable value, leave it
// null rather than fail the emission.
Guid? correlationId = Guid.TryParse(notificationId, out var parsed) ? parsed : (Guid?)null;
// Capture the request summary — {"subject": "...", "body": "..."}.
// Payload cap and per-target body redaction are applied at write
// time by the audit redactor (AuditLogOptions / PerTargetRedactionOverride).
var requestSummary = JsonSerializer.Serialize(new
{
subject = subject,
body = body,
});
evt = ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.Notification,
kind: AuditKind.NotifySend,
status: AuditStatus.Submitted,
occurredAtUtc: DateTime.SpecifyKind(occurredAtUtc, DateTimeKind.Utc),
// Outbound channel: per the Audit Log Actor-column spec the
// actor is the calling script. Null when no single script
// owns the call (e.g. a shared script running inline).
actor: _sourceScript,
target: _listName,
// CorrelationId is the NotificationId-derived per-operation
// lifecycle id; ExecutionId carries the per-execution id.
correlationId: correlationId,
executionId: _executionId,
// (ParentExecutionId): the spawning
// execution's id; null for non-routed runs.
parentExecutionId: _parentExecutionId,
sourceSiteId: string.IsNullOrEmpty(_siteId) ? null : _siteId,
sourceInstanceId: _instanceName,
sourceScript: _sourceScript,
httpStatus: null,
// Send is fire-and-forget from the script's perspective —
// the dispatcher (NotificationOutboxActor) times each
// delivery attempt and stamps DurationMs on its
// NotifyDeliver(Attempted) rows.
durationMs: null,
errorMessage: null,
errorDetail: null,
requestSummary: requestSummary,
responseSummary: null,
payloadTruncated: false,
extra: null);
}
catch (Exception buildEx)
{
// Defensive: building the event itself must never propagate.
_logger.LogWarning(buildEx,
"Failed to build Audit Log #23 NotifySend event for NotificationId {NotificationId} list '{List}' — skipping emission",
notificationId, _listName);
return;
}
try
{
// Flow Debugger: collector first (same instance), then the writer.
_traceSink?.Invoke(evt);
// Fire-and-forget (mirrors ExternalSystemHelper.EmitCallAudit)
// so the script is never blocked on the audit writer; we observe
// failures via ContinueWith so a thrown writer is logged rather
// than going to the unobserved-task firehose.
var writeTask = _auditWriter.WriteAsync(evt, CancellationToken.None);
if (!writeTask.IsCompleted)
{
writeTask.ContinueWith(
t => _logger.LogWarning(t.Exception,
"Audit Log #23 write failed for EventId {EventId} (NotifySend NotificationId {NotificationId})",
evt.EventId, notificationId),
CancellationToken.None,
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default);
}
else if (writeTask.IsFaulted)
{
_logger.LogWarning(writeTask.Exception,
"Audit Log #23 write failed for EventId {EventId} (NotifySend NotificationId {NotificationId})",
evt.EventId, notificationId);
}
}
catch (Exception writeEx)
{
// Synchronous throw from WriteAsync (e.g. ArgumentNullException
// before the writer's own try/catch). Swallow + log per alog.md §7.
_logger.LogWarning(writeEx,
"Audit Log #23 write threw synchronously for EventId {EventId} (NotifySend NotificationId {NotificationId})",
evt.EventId, notificationId);
}
}
}
///
/// Script-side accessor for cached-operation tracking.
/// Tracking.Status(trackedOperationId) reads the site-local SQLite
/// row directly via —
/// the site is the single source of truth for cached-call status, so no
/// central round-trip is needed and the call is answered authoritatively.
///
public class TrackingHelper
{
private readonly IOperationTrackingStore? _store;
private readonly ILogger _logger;
///
/// Initializes a new operation tracking helper.
///
/// Optional store for tracking cached operation status.
/// Logger for diagnostics and warnings.
internal TrackingHelper(IOperationTrackingStore? store, ILogger logger)
{
_store = store;
_logger = logger;
}
///
/// Returns the latest tracking snapshot for the supplied id, or
/// null when the id is unknown (never recorded, or purged after
/// the retention window).
///
/// The tracked operation identifier to query.
/// Cancellation token for the async operation.
/// The tracking status snapshot, or null if the operation is not found.
///
/// Thrown when the script runtime was constructed without an
/// — mirrors the
/// "service-not-wired" failure mode of the other integration helpers.
///
public Task Status(
TrackedOperationId trackedOperationId,
CancellationToken cancellationToken = default)
{
if (_store == null)
{
throw new InvalidOperationException(
"Operation tracking store not available");
}
return _store.GetStatusAsync(trackedOperationId, cancellationToken);
}
}
}