Joseph Doherty
703 lines
37 KiB
C#
703 lines
37 KiB
C#
using System.Diagnostics;
|
|
using System.Text.Json;
|
|
using Akka.Actor;
|
|
using Microsoft.Extensions.Logging;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
|
|
|
|
/// <summary>
|
|
/// C3/N3: a routed call failed because the target site never answered — the routed
|
|
/// Ask <b>expired</b>. Unreachability is typed at the Ask boundary from
|
|
/// <see cref="AskTimeoutException"/> (the communication layer's own signal: an
|
|
/// unreachable/contactless site has its enveloped message dropped so the Ask times
|
|
/// out caller-side) — it is <em>never</em> inferred from an error-message substring.
|
|
/// Derives from <see cref="InvalidOperationException"/> so existing callers that
|
|
/// catch the base type are unaffected, while <see cref="InboundScriptExecutor"/>
|
|
/// catches this more specific type to surface the spec's <c>SITE_UNREACHABLE</c>
|
|
/// error code instead of a generic <c>SCRIPT_ERROR</c>. A <c>Success=false</c>
|
|
/// <em>response</em> means the site answered (reachable by definition) and stays a
|
|
/// plain <see cref="InvalidOperationException"/>.
|
|
/// </summary>
|
|
public sealed class SiteUnreachableException : InvalidOperationException
|
|
{
|
|
/// <summary>Initializes a new <see cref="SiteUnreachableException"/> with the given message.</summary>
|
|
/// <param name="message">The routing-level failure message that indicated the site was unreachable.</param>
|
|
public SiteUnreachableException(string message) : base(message) { }
|
|
|
|
/// <summary>Initializes a new <see cref="SiteUnreachableException"/> wrapping the underlying Ask-timeout.</summary>
|
|
/// <param name="message">The routing-level failure message.</param>
|
|
/// <param name="innerException">The underlying <see cref="AskTimeoutException"/> from the routed Ask.</param>
|
|
public SiteUnreachableException(string message, Exception innerException) : base(message, innerException) { }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Arch-review F2: a routed call whose target site is outside the caller's site
|
|
/// scope. Raised by <c>RouteTarget.ResolveSiteAsync</c> — the single choke point every routed
|
|
/// operation passes through — so it is thrown <b>before any site traffic</b>: nothing is sent,
|
|
/// nothing is read, nothing is written.
|
|
///
|
|
/// <para>
|
|
/// The scope exists for the Flow Debugger's method-entry Live run, where an operator picks a
|
|
/// method rather than a site: a site-scoped Deployer must not reach outside their sites just
|
|
/// because the method's script routes there. The ordinary HTTP inbound path attaches no scope
|
|
/// (an API key is not a site-scoped principal) and is unaffected.
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// Derives from <see cref="InvalidOperationException"/> so existing callers catching the base
|
|
/// type are unaffected, while <see cref="InboundScriptExecutor"/> catches this more specific
|
|
/// type to surface the <c>SITE_SCOPE_DENIED</c> error code instead of a generic
|
|
/// <c>SCRIPT_ERROR</c>. Deliberately NOT the ManagementService's <c>SiteScopeViolationException</c>
|
|
/// — InboundAPI cannot reference ManagementService; the runner converts the code back.
|
|
/// </para>
|
|
/// </summary>
|
|
public sealed class RouteSiteScopeException : InvalidOperationException
|
|
{
|
|
/// <summary>Initializes a new <see cref="RouteSiteScopeException"/> with the given message.</summary>
|
|
/// <param name="message">The scope-denial message naming the instance and its site.</param>
|
|
public RouteSiteScopeException(string message) : base(message) { }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Route.To() helper for cross-site calls from inbound API scripts.
|
|
/// Resolves instance to site, routes via <see cref="IInstanceRouter"/>, blocks until
|
|
/// response or timeout. Site unreachable returns error (no store-and-forward).
|
|
///
|
|
/// The helper carries the executing method's <see cref="CancellationToken"/>
|
|
/// (the method-level timeout). Routed calls inherit that deadline by default, so a
|
|
/// natural script — <c>Route.To("inst").Call("doWork", p)</c> — is timeout-bounded
|
|
/// without the script having to thread a token explicitly.
|
|
/// </summary>
|
|
public class RouteHelper
|
|
{
|
|
private readonly IInstanceLocator _instanceLocator;
|
|
private readonly IInstanceRouter _instanceRouter;
|
|
private readonly CancellationToken _deadlineToken;
|
|
private readonly CancellationToken _requestAbortedToken;
|
|
private readonly Guid? _parentExecutionId;
|
|
private readonly ICentralAuditWriter? _auditWriter;
|
|
private readonly INodeIdentityProvider? _nodeIdentity;
|
|
private readonly bool _trace;
|
|
private readonly Action<AuditEvent>? _traceSink;
|
|
private readonly IReadOnlySet<string>? _permittedSiteIdentifiers;
|
|
private readonly bool _captureBodies;
|
|
private readonly ILogger? _logger;
|
|
|
|
/// <summary>
|
|
/// Initializes a new <see cref="RouteHelper"/> with no deadline token and no parent execution id.
|
|
/// DI (<c>AddScoped<RouteHelper>()</c>) resolves the two optional audit collaborators when the
|
|
/// host registers them; a bare <c>new RouteHelper(locator, router)</c> writes no audit rows.
|
|
/// </summary>
|
|
/// <param name="instanceLocator">Service to resolve the site id for a given instance code.</param>
|
|
/// <param name="instanceRouter">Service to route cross-site calls to the resolved site.</param>
|
|
/// <param name="auditWriter">Flow Debugger (design 2026-09-05 §5.1): central-direct-write audit writer for the always-on <c>RouteCall</c> row; null = no row.</param>
|
|
/// <param name="nodeIdentity">Stamps <c>SourceNode</c> (central-a/central-b) on the <c>RouteCall</c> row; null = NULL.</param>
|
|
/// <param name="logger">Arch-review F8: logs a Warning when the best-effort <c>RouteCall</c> audit write faults; null = silent.</param>
|
|
public RouteHelper(
|
|
IInstanceLocator instanceLocator,
|
|
IInstanceRouter instanceRouter,
|
|
ICentralAuditWriter? auditWriter = null,
|
|
INodeIdentityProvider? nodeIdentity = null,
|
|
ILogger<RouteHelper>? logger = null)
|
|
: this(instanceLocator, instanceRouter, CancellationToken.None, CancellationToken.None, parentExecutionId: null,
|
|
auditWriter, nodeIdentity, trace: false, traceSink: null, permittedSiteIdentifiers: null, captureBodies: true, logger)
|
|
{
|
|
}
|
|
|
|
private RouteHelper(
|
|
IInstanceLocator instanceLocator,
|
|
IInstanceRouter instanceRouter,
|
|
CancellationToken deadlineToken,
|
|
CancellationToken requestAbortedToken,
|
|
Guid? parentExecutionId,
|
|
ICentralAuditWriter? auditWriter,
|
|
INodeIdentityProvider? nodeIdentity,
|
|
bool trace,
|
|
Action<AuditEvent>? traceSink,
|
|
IReadOnlySet<string>? permittedSiteIdentifiers,
|
|
bool captureBodies,
|
|
ILogger? logger)
|
|
{
|
|
_permittedSiteIdentifiers = permittedSiteIdentifiers;
|
|
_instanceLocator = instanceLocator;
|
|
_instanceRouter = instanceRouter;
|
|
_deadlineToken = deadlineToken;
|
|
_requestAbortedToken = requestAbortedToken;
|
|
_parentExecutionId = parentExecutionId;
|
|
_auditWriter = auditWriter;
|
|
_nodeIdentity = nodeIdentity;
|
|
_trace = trace;
|
|
_traceSink = traceSink;
|
|
_captureBodies = captureBodies;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a <see cref="RouteHelper"/> whose routed calls inherit
|
|
/// <paramref name="deadlineToken"/> (the executing method's timeout) by default.
|
|
/// <see cref="InboundScriptExecutor"/> calls this when it builds the script
|
|
/// context so the method timeout actually covers routed calls, as the design doc
|
|
/// requires.
|
|
/// </summary>
|
|
/// <param name="deadlineToken">The executing method's timeout cancellation token to inherit for routed calls.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> inheriting the given deadline token.</returns>
|
|
public RouteHelper WithDeadline(CancellationToken deadlineToken) =>
|
|
new(_instanceLocator, _instanceRouter, deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Returns a <see cref="RouteHelper"/> carrying the raw request-abort
|
|
/// token (a client disconnect) <em>separately</em> from the method deadline. Most
|
|
/// routed calls remain bounded by the method deadline (which already incorporates the
|
|
/// abort), but <see cref="RouteTarget.WaitForAttribute"/> uses this token so its wait
|
|
/// is bounded by the WAIT timeout — not the generic method deadline (spec §6) — while
|
|
/// still being cancelled by a client disconnect.
|
|
/// </summary>
|
|
/// <param name="requestAbortedToken">The raw client-disconnect token, independent of the method timeout.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the given request-abort token.</returns>
|
|
public RouteHelper WithRequestAborted(CancellationToken requestAbortedToken) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Audit Log (ParentExecutionId): returns a <see cref="RouteHelper"/> whose
|
|
/// routed <see cref="RouteTarget.Call"/> requests carry
|
|
/// <paramref name="parentExecutionId"/> as <see cref="RouteToCallRequest.ParentExecutionId"/>.
|
|
/// For an inbound API request this is the inbound request's own per-request
|
|
/// execution id, so the routed site script records the inbound request as its
|
|
/// parent. <see cref="InboundScriptExecutor"/> calls this when it builds the
|
|
/// script context.
|
|
/// </summary>
|
|
/// <param name="parentExecutionId">The inbound request's execution id to stamp as the spawning parent on routed calls, or null for non-routed runs.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the given parent execution id.</returns>
|
|
public RouteHelper WithParentExecutionId(Guid? parentExecutionId) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Flow Debugger (design 2026-09-05 §5.2): returns a <see cref="RouteHelper"/> whose routed
|
|
/// calls and writes carry <c>Trace = <paramref name="trace"/></c>, so the site traces the
|
|
/// spawned run. The always-on <c>RouteCall</c> row is written regardless of this flag.
|
|
/// </summary>
|
|
/// <param name="trace">True to trace the routed site runs.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the flag.</returns>
|
|
public RouteHelper WithTrace(bool trace) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, trace, _traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Flow Debugger (design 2026-09-05 §7.1): returns a <see cref="RouteHelper"/> that also
|
|
/// hands every <c>RouteCall</c> row AND every inline site trace event on the routed reply
|
|
/// to <paramref name="traceSink"/>, so a Live run can render without a round-trip to the
|
|
/// audit store. Null clears the sink.
|
|
/// </summary>
|
|
/// <param name="traceSink">Receives the audit events in emission order; null = none.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the sink.</returns>
|
|
public RouteHelper WithTraceSink(Action<AuditEvent>? traceSink) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Flow Debugger outbound calls (plan 2026-09-06, D5/D7): the trace sink this helper carries,
|
|
/// so <see cref="InboundScriptExecutor"/> can hand the SAME delegate to the script's
|
|
/// <see cref="InboundDatabaseHelper"/> and a Live run's central <c>Database.*</c> rows land
|
|
/// in the same collector as its <c>RouteCall</c> rows. Null when no sink was attached.
|
|
/// </summary>
|
|
public Action<AuditEvent>? TraceSink => _traceSink;
|
|
|
|
/// <summary>
|
|
/// Arch-review F2: returns a <see cref="RouteHelper"/> that refuses a routed call
|
|
/// whose target site is not in <paramref name="permittedSiteIdentifiers"/> — site
|
|
/// <em>identifiers</em> (e.g. <c>site-a</c>), the values <see cref="IInstanceLocator"/> returns,
|
|
/// NOT the numeric ids an <c>AuthenticatedUser.PermittedSiteIds</c> carries. Matching is
|
|
/// whatever comparer the supplied set uses, so pass an ordinal-ignore-case set.
|
|
///
|
|
/// <para>
|
|
/// <c>null</c> = unscoped, the default and the state of every ordinary inbound HTTP request.
|
|
/// An EMPTY set is not the same thing: it permits nothing, which is the correct outcome for
|
|
/// a site-scoped user whose sites no longer exist.
|
|
/// </para>
|
|
///
|
|
/// <para>
|
|
/// The Flow Debugger's method-entry Live run sets this from the operator's site-scoped
|
|
/// Deployer role, because the operator picks a <em>method</em> — the site is only known once
|
|
/// the script routes, so the check cannot happen up front the way it does for a value-change
|
|
/// run's named instance.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <param name="permittedSiteIdentifiers">The site identifiers routed calls may reach; null = unscoped.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the scope.</returns>
|
|
public RouteHelper WithSiteScope(IReadOnlySet<string>? permittedSiteIdentifiers) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, permittedSiteIdentifiers, _captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Arch-review F11: returns a <see cref="RouteHelper"/> whose always-on <c>RouteCall</c> row
|
|
/// carries no request/response summary when <paramref name="captureBodies"/> is <c>false</c>.
|
|
/// <see cref="InboundScriptExecutor"/> sets this from the executing method's
|
|
/// <c>AuditLogOptions.PerTargetOverrides[method.Name].SkipBodyCapture</c> — the same override
|
|
/// <c>AuditWriteMiddleware</c> honours for the inbound row — so an operator who suppressed a
|
|
/// method's body from the audit log doesn't get it back via the routed call's arguments
|
|
/// (which are derived from that same body). Default <c>true</c> (unchanged behaviour).
|
|
/// </summary>
|
|
/// <param name="captureBodies">False to omit the request/response summary on the <c>RouteCall</c> row.</param>
|
|
/// <returns>A new <see cref="RouteHelper"/> carrying the flag.</returns>
|
|
public RouteHelper WithBodyCapture(bool captureBodies) =>
|
|
new(_instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, _permittedSiteIdentifiers, captureBodies, _logger);
|
|
|
|
/// <summary>
|
|
/// Creates a route target for the specified instance.
|
|
/// </summary>
|
|
/// <param name="instanceCode">The unique code of the instance to route calls to.</param>
|
|
/// <returns>A <see cref="RouteTarget"/> bound to the specified instance.</returns>
|
|
public RouteTarget To(string instanceCode)
|
|
{
|
|
return new RouteTarget(
|
|
instanceCode, _instanceLocator, _instanceRouter, _deadlineToken, _requestAbortedToken, _parentExecutionId,
|
|
_auditWriter, _nodeIdentity, _trace, _traceSink, _permittedSiteIdentifiers, _captureBodies, _logger);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Represents a route target (an instance) for cross-site calls.
|
|
/// </summary>
|
|
public class RouteTarget
|
|
{
|
|
private readonly string _instanceCode;
|
|
private readonly IInstanceLocator _instanceLocator;
|
|
private readonly IInstanceRouter _instanceRouter;
|
|
private readonly CancellationToken _deadlineToken;
|
|
private readonly CancellationToken _requestAbortedToken;
|
|
private readonly Guid? _parentExecutionId;
|
|
private readonly ICentralAuditWriter? _auditWriter;
|
|
private readonly INodeIdentityProvider? _nodeIdentity;
|
|
private readonly bool _trace;
|
|
private readonly Action<AuditEvent>? _traceSink;
|
|
private readonly IReadOnlySet<string>? _permittedSiteIdentifiers;
|
|
private readonly bool _captureBodies;
|
|
private readonly ILogger? _logger;
|
|
|
|
/// <summary>
|
|
/// Initializes a new <see cref="RouteTarget"/> for the specified instance.
|
|
/// </summary>
|
|
/// <param name="instanceCode">The unique code of the target instance.</param>
|
|
/// <param name="instanceLocator">Service to resolve the site id for the instance.</param>
|
|
/// <param name="instanceRouter">Service to route cross-site calls.</param>
|
|
/// <param name="deadlineToken">Cancellation token representing the method-level deadline.</param>
|
|
/// <param name="requestAbortedToken">Raw client-disconnect token, independent of the method timeout.</param>
|
|
/// <param name="parentExecutionId">Optional parent execution id for audit correlation on routed calls.</param>
|
|
/// <param name="auditWriter">Central-direct-write audit writer for the always-on <c>RouteCall</c> row; null = no row.</param>
|
|
/// <param name="nodeIdentity">Stamps <c>SourceNode</c> on the <c>RouteCall</c> row; null = NULL.</param>
|
|
/// <param name="trace">True to trace the routed site run.</param>
|
|
/// <param name="traceSink">Receives the emitted audit events in order; null = none.</param>
|
|
/// <param name="permittedSiteIdentifiers">Arch-review F2: site identifiers this call may reach; null = unscoped.</param>
|
|
/// <param name="captureBodies">Arch-review F11: false omits the request/response summary on the <c>RouteCall</c> row.</param>
|
|
/// <param name="logger">Arch-review F8: logs a Warning when the best-effort <c>RouteCall</c> audit write faults; null = silent.</param>
|
|
internal RouteTarget(
|
|
string instanceCode,
|
|
IInstanceLocator instanceLocator,
|
|
IInstanceRouter instanceRouter,
|
|
CancellationToken deadlineToken,
|
|
CancellationToken requestAbortedToken,
|
|
Guid? parentExecutionId,
|
|
ICentralAuditWriter? auditWriter = null,
|
|
INodeIdentityProvider? nodeIdentity = null,
|
|
bool trace = false,
|
|
Action<AuditEvent>? traceSink = null,
|
|
IReadOnlySet<string>? permittedSiteIdentifiers = null,
|
|
bool captureBodies = true,
|
|
ILogger? logger = null)
|
|
{
|
|
_permittedSiteIdentifiers = permittedSiteIdentifiers;
|
|
_instanceCode = instanceCode;
|
|
_instanceLocator = instanceLocator;
|
|
_instanceRouter = instanceRouter;
|
|
_deadlineToken = deadlineToken;
|
|
_requestAbortedToken = requestAbortedToken;
|
|
_parentExecutionId = parentExecutionId;
|
|
_auditWriter = auditWriter;
|
|
_nodeIdentity = nodeIdentity;
|
|
_trace = trace;
|
|
_traceSink = traceSink;
|
|
_captureBodies = captureBodies;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calls a script on the remote instance. Synchronous from API caller's
|
|
/// perspective. <paramref name="parameters"/> may be a dictionary or an
|
|
/// anonymous object (<c>new { name = "Bob" }</c>) — see <see cref="ScriptArgs"/>.
|
|
///
|
|
/// When <paramref name="cancellationToken"/> is not supplied the
|
|
/// routed call inherits the executing method's timeout, so the call is bounded by
|
|
/// the method-level deadline with no token argument.
|
|
/// </summary>
|
|
/// <param name="scriptName">Name of the script to call on the remote instance.</param>
|
|
/// <param name="parameters">Optional parameters passed to the script; may be a dictionary or anonymous object.</param>
|
|
/// <param name="cancellationToken">Optional cancellation token; defaults to the method deadline when not supplied.</param>
|
|
/// <returns>A task that resolves to the remote script's return value, or null.</returns>
|
|
public async Task<object?> Call(
|
|
string scriptName,
|
|
object? parameters = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var token = Effective(cancellationToken);
|
|
var siteId = await ResolveSiteAsync(token);
|
|
var correlationId = Guid.NewGuid().ToString();
|
|
|
|
// Audit Log (ParentExecutionId): stamp the spawning execution's id
|
|
// (the inbound API request's ExecutionId) so the routed site script
|
|
// records this call's parent. CorrelationId above is a separate concern
|
|
// — the per-operation lifecycle id, freshly minted per routed call.
|
|
var normalized = ScriptArgs.Normalize(parameters);
|
|
var request = new RouteToCallRequest(
|
|
correlationId, _instanceCode, scriptName, normalized,
|
|
DateTimeOffset.UtcNow, _parentExecutionId,
|
|
// Flow Debugger (design 2026-09-05 §5.2): the trace flag rides beside the parent id.
|
|
Trace: _trace);
|
|
|
|
// Flow Debugger (design §5.1): the ALWAYS-ON RouteCall row — one per routed call,
|
|
// written whether or not the run is traced (a writer must be present).
|
|
var startedUtc = DateTime.UtcNow;
|
|
var sw = Stopwatch.StartNew();
|
|
RouteToCallResponse response;
|
|
try
|
|
{
|
|
response = await RouteOrUnreachable(_instanceRouter.RouteToCallAsync(siteId, request, token));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
EmitRouteCall(scriptName, normalized, startedUtc, sw.ElapsedMilliseconds, success: false,
|
|
returnValue: null, error: ex.Message, childExecutionId: null, traceEvents: null);
|
|
throw;
|
|
}
|
|
|
|
EmitRouteCall(scriptName, normalized, startedUtc, sw.ElapsedMilliseconds, response.Success,
|
|
response.ReturnValue, response.ErrorMessage, response.ChildExecutionId, response.TraceEvents);
|
|
|
|
if (!response.Success)
|
|
{
|
|
throw RoutingFailure(
|
|
response.ErrorMessage ?? "Remote script call failed");
|
|
}
|
|
|
|
return response.ReturnValue;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Flow Debugger (design 2026-09-05 §5.1/§7.1): builds the <c>ScriptTrace.RouteCall</c> row,
|
|
/// writes it best-effort through the central audit writer (never throws into the script),
|
|
/// and hands it — followed by the site's inline trace events — to the trace sink.
|
|
/// </summary>
|
|
private void EmitRouteCall(
|
|
string scriptName, IReadOnlyDictionary<string, object?>? args, DateTime startedUtc, long elapsedMs,
|
|
bool success, object? returnValue, string? error, Guid? childExecutionId,
|
|
IReadOnlyList<AuditEvent>? traceEvents)
|
|
{
|
|
if (_auditWriter is null && _traceSink is null) return;
|
|
AuditEvent evt;
|
|
try
|
|
{
|
|
evt = ScadaBridgeAuditEventFactory.Create(
|
|
AuditChannel.ScriptTrace, AuditKind.RouteCall,
|
|
success ? AuditStatus.Delivered : AuditStatus.Failed,
|
|
occurredAtUtc: DateTime.SpecifyKind(startedUtc, DateTimeKind.Utc),
|
|
actor: "inbound-api",
|
|
target: $"{_instanceCode}.{scriptName}",
|
|
sourceNode: _nodeIdentity?.NodeName,
|
|
executionId: _parentExecutionId,
|
|
parentExecutionId: null,
|
|
durationMs: (int)Math.Min(elapsedMs, int.MaxValue),
|
|
errorMessage: error,
|
|
// Arch-review F11: a method with SkipBodyCapture set must not leak the routed
|
|
// call's arguments (derived from the same request body) or its return value.
|
|
requestSummary: _captureBodies ? TraceJson(args) : null,
|
|
responseSummary: _captureBodies ? TraceJson(returnValue) : null,
|
|
extra: childExecutionId is null ? null : new ScriptTraceExtra { ChildExecutionId = childExecutionId }.ToJson());
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (_auditWriter is not null)
|
|
{
|
|
try
|
|
{
|
|
var t = _auditWriter.WriteAsync(evt, CancellationToken.None);
|
|
if (!t.IsCompletedSuccessfully)
|
|
{
|
|
_ = t.ContinueWith(
|
|
x => _logger?.LogWarning(x.Exception, "Flow trace RouteCall write failed for {Target}", $"{_instanceCode}.{scriptName}"),
|
|
CancellationToken.None,
|
|
TaskContinuationOptions.OnlyOnFaulted | TaskContinuationOptions.ExecuteSynchronously,
|
|
TaskScheduler.Default);
|
|
}
|
|
}
|
|
catch (Exception)
|
|
{
|
|
// Best-effort: an audit failure never fails the routed call.
|
|
}
|
|
}
|
|
|
|
if (_traceSink is null) return;
|
|
try
|
|
{
|
|
_traceSink(evt);
|
|
if (traceEvents is not null)
|
|
{
|
|
foreach (var e in traceEvents) _traceSink(e);
|
|
}
|
|
}
|
|
catch (Exception)
|
|
{
|
|
// A misbehaving sink must not fail the script either.
|
|
}
|
|
}
|
|
|
|
private static string? TraceJson(object? value)
|
|
{
|
|
if (value is null) return null;
|
|
try { return JsonSerializer.Serialize(value, ScriptTraceExtra.JsonOptions); }
|
|
catch (Exception) { return null; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets a single attribute value from the remote instance.
|
|
/// </summary>
|
|
/// <param name="attributeName">Name of the attribute to read.</param>
|
|
/// <param name="cancellationToken">Optional cancellation token; defaults to the method deadline.</param>
|
|
/// <returns>A task that resolves to the attribute value, or null if the key is absent.</returns>
|
|
public async Task<object?> GetAttribute(
|
|
string attributeName,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var result = await GetAttributes(new[] { attributeName }, cancellationToken);
|
|
return result.TryGetValue(attributeName, out var value) ? value : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets multiple attribute values from the remote instance (batch read).
|
|
/// </summary>
|
|
/// <param name="attributeNames">Names of the attributes to read.</param>
|
|
/// <param name="cancellationToken">Optional cancellation token; defaults to the method deadline.</param>
|
|
/// <returns>A task that resolves to a map of attribute names to their current values.</returns>
|
|
public async Task<IReadOnlyDictionary<string, object?>> GetAttributes(
|
|
IEnumerable<string> attributeNames,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var token = Effective(cancellationToken);
|
|
var siteId = await ResolveSiteAsync(token);
|
|
var correlationId = Guid.NewGuid().ToString();
|
|
|
|
// Audit Log (ParentExecutionId): mirrors the Call path — stamp the
|
|
// spawning inbound request's ExecutionId so future site-side audit
|
|
// emission for routed reads can record this read's parent. Symmetric
|
|
// with RouteToCallRequest so script authors get the same correlation
|
|
// across Call / GetAttributes / SetAttributes.
|
|
var request = new RouteToGetAttributesRequest(
|
|
correlationId, _instanceCode, attributeNames.ToList(), DateTimeOffset.UtcNow,
|
|
_parentExecutionId);
|
|
|
|
var response = await RouteOrUnreachable(_instanceRouter.RouteToGetAttributesAsync(siteId, request, token));
|
|
|
|
if (!response.Success)
|
|
{
|
|
throw RoutingFailure(
|
|
response.ErrorMessage ?? "Remote attribute read failed");
|
|
}
|
|
|
|
return response.Values;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Blocks until a remote instance attribute reaches <paramref name="targetValue"/>
|
|
/// or <paramref name="timeout"/> elapses (spec §6). Value-equality ONLY across the
|
|
/// wire: the target is canonically encoded via <see cref="AttributeValueCodec"/> and
|
|
/// the site evaluates equality — there is no predicate and no quality flag in the
|
|
/// comparison.
|
|
///
|
|
/// <para>
|
|
/// Unlike the other routed calls (which inherit the method deadline
|
|
/// by default), the wait is bounded by <paramref name="timeout"/> — the WAIT
|
|
/// timeout — NOT the generic method deadline. The SITE enforces <paramref name="timeout"/>
|
|
/// and returns <c>Matched=false</c> when it elapses; the local token is built from a
|
|
/// per-wait CTS (the wait timeout plus a small grace), an explicit caller token, and
|
|
/// the client-disconnect token, but deliberately EXCLUDES the method deadline. So a
|
|
/// wait longer than the method timeout runs to its full wait timeout, as spec §6
|
|
/// requires, while a client disconnect still cancels it.
|
|
/// </para>
|
|
/// </summary>
|
|
/// <param name="attributeName">Name of the attribute to wait on.</param>
|
|
/// <param name="targetValue">Target value the attribute must equal for the wait to match.</param>
|
|
/// <param name="timeout">Maximum time to wait for the attribute to reach the target value; this — not the method deadline — bounds the wait.</param>
|
|
/// <param name="cancellationToken">Optional explicit cancellation token (a tighter caller-supplied bound); the wait is otherwise bounded by <paramref name="timeout"/>.</param>
|
|
/// <returns>A task that resolves to <c>true</c> if the attribute reached the target value, <c>false</c> if the wait timed out.</returns>
|
|
public async Task<bool> WaitForAttribute(
|
|
string attributeName,
|
|
object? targetValue,
|
|
TimeSpan timeout,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
// Do NOT impose a local wait-timeout backstop here. The site
|
|
// enforces the wait `timeout` and returns Matched=false when it elapses, and the
|
|
// cluster Ask in CommunicationService.RouteToWaitForAttributeAsync already bounds
|
|
// the round trip by `timeout + IntegrationTimeout` — the authoritative backstop
|
|
// for a missing site response. A local CTS of `timeout + small grace` (the prior
|
|
// approach) was TIGHTER than that round-trip budget, so a
|
|
// slow-but-valid timed-out response could be cancelled into an exception instead
|
|
// of the spec-mandated `false`. Link ONLY the client-disconnect token and an
|
|
// explicit caller token — NOT the method deadline — so a client abort still
|
|
// cancels the wait while the wait timeout itself governs the duration.
|
|
using var linked = CancellationTokenSource.CreateLinkedTokenSource(
|
|
_requestAbortedToken, cancellationToken);
|
|
var token = linked.Token;
|
|
var siteId = await ResolveSiteAsync(token);
|
|
|
|
// Audit Log (ParentExecutionId): mirrors the Call path — stamp the
|
|
// spawning inbound request's ExecutionId so future site-side audit
|
|
// emission for routed waits can record this wait's parent. CorrelationId
|
|
// is the per-operation lifecycle id, freshly minted per routed wait.
|
|
var request = new RouteToWaitForAttributeRequest(
|
|
Guid.NewGuid().ToString(), _instanceCode, attributeName,
|
|
AttributeValueCodec.Encode(targetValue), timeout, DateTimeOffset.UtcNow,
|
|
_parentExecutionId);
|
|
|
|
var response = await RouteOrUnreachable(_instanceRouter.RouteToWaitForAttributeAsync(siteId, request, token));
|
|
|
|
if (!response.Success)
|
|
{
|
|
throw RoutingFailure(
|
|
response.ErrorMessage ?? "Remote attribute wait failed");
|
|
}
|
|
|
|
return response.Matched;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sets a single attribute value on the remote instance.
|
|
/// </summary>
|
|
/// <param name="attributeName">Name of the attribute to write.</param>
|
|
/// <param name="value">Value to set on the attribute.</param>
|
|
/// <param name="cancellationToken">Optional cancellation token; defaults to the method deadline.</param>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
public async Task SetAttribute(
|
|
string attributeName,
|
|
string value,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
await SetAttributes(
|
|
new Dictionary<string, string> { { attributeName, value } },
|
|
cancellationToken);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Sets multiple attribute values on the remote instance (batch write).
|
|
/// </summary>
|
|
/// <param name="attributeValues">Map of attribute names to values to write.</param>
|
|
/// <param name="cancellationToken">Optional cancellation token; defaults to the method deadline.</param>
|
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
|
public async Task SetAttributes(
|
|
IReadOnlyDictionary<string, string> attributeValues,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var token = Effective(cancellationToken);
|
|
var siteId = await ResolveSiteAsync(token);
|
|
var correlationId = Guid.NewGuid().ToString();
|
|
|
|
// Audit Log (ParentExecutionId): mirrors the Call path — stamp the
|
|
// spawning inbound request's ExecutionId so future site-side audit
|
|
// emission for routed writes can record this write's parent. Symmetric
|
|
// with RouteToCallRequest so script authors get the same correlation
|
|
// across Call / GetAttributes / SetAttributes.
|
|
var request = new RouteToSetAttributesRequest(
|
|
correlationId, _instanceCode, attributeValues, DateTimeOffset.UtcNow,
|
|
_parentExecutionId,
|
|
// Flow Debugger (design §5.3): a traced routed write arms the site's echo stamp.
|
|
Trace: _trace);
|
|
|
|
var response = await RouteOrUnreachable(_instanceRouter.RouteToSetAttributesAsync(siteId, request, token));
|
|
|
|
if (!response.Success)
|
|
{
|
|
throw RoutingFailure(
|
|
response.ErrorMessage ?? "Remote attribute write failed");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// A routed call with no explicit token inherits the executing
|
|
/// method's deadline. An explicitly supplied token (for a tighter bound) wins.
|
|
/// </summary>
|
|
private CancellationToken Effective(CancellationToken explicitToken) =>
|
|
explicitToken.CanBeCanceled ? explicitToken : _deadlineToken;
|
|
|
|
private async Task<string> ResolveSiteAsync(CancellationToken cancellationToken)
|
|
{
|
|
var siteId = await _instanceLocator.GetSiteIdForInstanceAsync(_instanceCode, cancellationToken);
|
|
if (siteId == null)
|
|
{
|
|
// A null site id is a plain not-found / no-assigned-site — never
|
|
// unreachability (nothing was routed), so it stays a generic
|
|
// InvalidOperationException (SCRIPT_ERROR).
|
|
throw RoutingFailure(
|
|
$"Instance '{_instanceCode}' not found or has no assigned site");
|
|
}
|
|
|
|
// Arch-review F2: the site scope gate. This method is the single choke point —
|
|
// Call, GetAttributes, WaitForAttribute and SetAttributes all resolve the site
|
|
// here first — so refusing at this point guarantees no site traffic of any kind
|
|
// is emitted for an out-of-scope instance.
|
|
if (_permittedSiteIdentifiers is not null && !_permittedSiteIdentifiers.Contains(siteId))
|
|
{
|
|
throw new RouteSiteScopeException(
|
|
$"Access denied: instance '{_instanceCode}' is at site '{siteId}', outside your site scope.");
|
|
}
|
|
|
|
return siteId;
|
|
}
|
|
|
|
/// <summary>
|
|
/// N3: awaits a routed Ask and TYPES unreachability. AskTimeoutException means
|
|
/// the site never answered (no route to the site, site down, partition) —
|
|
/// the communication layer's own typed signal (CentralCommunicationActor drops
|
|
/// envelopes for contactless sites so the Ask expires caller-side) — and
|
|
/// surfaces as SiteUnreachableException so the executor maps it to the spec's
|
|
/// SITE_UNREACHABLE code. A Success=false RESPONSE means the site answered and
|
|
/// is therefore reachable; it flows through RoutingFailure as a plain
|
|
/// InvalidOperationException (SCRIPT_ERROR).
|
|
/// </summary>
|
|
private static async Task<TResponse> RouteOrUnreachable<TResponse>(Task<TResponse> routedAsk)
|
|
{
|
|
try
|
|
{
|
|
return await routedAsk;
|
|
}
|
|
catch (AskTimeoutException ex)
|
|
{
|
|
throw new SiteUnreachableException(
|
|
"Site did not respond to the routed request within the routing timeout — unreachable or not connected.",
|
|
ex);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// C3/N3: a routing-level failure REPORTED BY the site or the locator. The
|
|
/// remote end answered (or the instance simply has no site), so this is never
|
|
/// unreachability — always a plain <see cref="InvalidOperationException"/>
|
|
/// (SCRIPT_ERROR at the endpoint). Unreachability is typed at the Ask boundary
|
|
/// by <c>RouteOrUnreachable</c> (AskTimeoutException → SiteUnreachableException),
|
|
/// replacing the old message-substring sniff that a harmless rewording in the
|
|
/// communication layer could silently defeat.
|
|
/// </summary>
|
|
private static InvalidOperationException RoutingFailure(string message) => new(message);
|
|
}
|