using System.Security.Claims;
using System.Text.Json;
using Akka.Actor;
using Microsoft.AspNetCore.Components.Authorization;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.CentralUI.Auth;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Templates;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Flattening;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Scripts;
using ZB.MOM.WW.ScadaBridge.Communication;
using ZB.MOM.WW.ScadaBridge.DeploymentManager;
using ZB.MOM.WW.ScadaBridge.ManagementService;
using ZB.MOM.WW.ScadaBridge.ManagementService.FlowDebug;
using ZB.MOM.WW.ScadaBridge.ScriptAnalysis;
using ZB.MOM.WW.ScadaBridge.Security;
namespace ZB.MOM.WW.ScadaBridge.CentralUI.Services;
///
/// Default : prediction runs in-process over the repositories
/// (method script + instance → template chain → script bodies, DB-only, nothing executes),
/// while the script entry (design 2026-09-08 §2) resolves an instance's scripts through the
/// DeploymentManager's so the picker, the prediction and the
/// run all see the same canonical names, trigger kinds and effective parameter definitions the
/// runner will;
/// the run/replay commands go to the per-node ManagementActor through
/// , exactly as
/// does, with the current Blazor principal projected to an
/// so the server's Deployer gate runs against the real identity. The Live-run Ask wait is
/// derived from the server's own bound ( /
/// ) rather than a fixed duration (arch-review F4) —
/// a run against a method with a long TimeoutSeconds used to be cut off client-side
/// well before the server's own grace period expired.
/// Retry / Discard of a parked outbound node (follow-ups plan 2026-09-06 D4) goes through
/// to the SAME central-local proxies the Site Calls and
/// Notification report pages use, so the page and the report never disagree on what a relay does.
///
public sealed class FlowDebugService : IFlowDebugService
{
/// Ask timeout for Replay (a central DB read).
public static readonly TimeSpan ReplayAskTimeout = TimeSpan.FromSeconds(30);
/// Slack the client adds over the server's own bound (Ask hop + serialization).
public static readonly TimeSpan ClientSlack = TimeSpan.FromSeconds(15);
/// Floor for a method-run Ask wait, regardless of how short the method's own timeout is.
public static readonly TimeSpan MinRunAskTimeout = TimeSpan.FromSeconds(30);
///
/// Client-side Ask wait for : the method's own
/// TimeoutSeconds plus the server's plus
/// , floored at — the server bounds
/// the run itself at timeoutSeconds + RunGrace (see FlowDebugRunner.RunMethodAsync),
/// so the client must wait at least that long or it cuts the run off before the server does.
///
public static TimeSpan RunAskTimeoutFor(int timeoutSeconds)
{
var t = TimeSpan.FromSeconds(Math.Max(1, timeoutSeconds)) + FlowDebugRunner.RunGrace + ClientSlack;
return t < MinRunAskTimeout ? MinRunAskTimeout : t;
}
///
/// Client-side Ask wait for a WRITE-kind script run (): the
/// trigger-wait window plus .
///
public static TimeSpan ValueChangeAskTimeoutFor(int waitSeconds)
=> TimeSpan.FromSeconds(Math.Max(1, waitSeconds)) + ClientSlack;
/// Upper bound on the template parent chain the script resolver walks (cycle guard).
private const int MaxTemplateChain = 16;
private static readonly JsonSerializerOptions ResultJsonOptions = new()
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
};
private readonly ManagementActorHolder _holder;
private readonly AuthenticationStateProvider _auth;
private readonly IInboundApiRepository _inboundApi;
private readonly ISiteRepository _sites;
private readonly ITemplateEngineRepository _templates;
private readonly IFlatteningPipeline _pipeline;
private readonly CommunicationService _comms;
private readonly ILogger _logger;
///
/// Flattened script lists by instance id. A scoped service in Blazor Server lives for the
/// whole CIRCUIT, so this is emphatically not a per-interaction memo: it is refreshed by
/// every call (the page makes one per instance
/// selection), and only the prediction and the run read it back. That keeps one flatten per
/// picker selection while making a re-selection pick up a template edit or a redeploy — a
/// cache held for the session would otherwise serve a stale parameter schema (unexplained
/// VALIDATION_FAILED) and a stale Ask bound against a runner that re-flattens on every
/// run. The raw s ride along because prediction needs the code
/// bodies, which the picker's projection deliberately does not carry.
///
private readonly Dictionary _scriptCache = new();
/// One instance's flattened scripts: the unique name the site keys on, the raw resolved scripts, and the picker projection.
private sealed record InstanceScripts(
string UniqueName,
IReadOnlyList Scripts,
IReadOnlyList Options);
/// Initializes the service.
public FlowDebugService(
ManagementActorHolder holder,
AuthenticationStateProvider auth,
IInboundApiRepository inboundApi,
ITemplateEngineRepository templates,
IFlatteningPipeline pipeline,
CommunicationService comms,
ILogger logger,
ISiteRepository sites)
{
_sites = sites ?? throw new ArgumentNullException(nameof(sites));
_holder = holder ?? throw new ArgumentNullException(nameof(holder));
_auth = auth ?? throw new ArgumentNullException(nameof(auth));
_inboundApi = inboundApi ?? throw new ArgumentNullException(nameof(inboundApi));
_templates = templates ?? throw new ArgumentNullException(nameof(templates));
_pipeline = pipeline ?? throw new ArgumentNullException(nameof(pipeline));
_comms = comms ?? throw new ArgumentNullException(nameof(comms));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
///
/// Upper bound on the resolve-and-rewalk passes of . Each pass can
/// only discover targets one nesting level deeper than the last, so the predictor's own depth cap
/// bounds the useful pass count; the constant is a safety net, not a tuning knob.
///
internal const int MaxPredictionPasses = 8;
///
public async Task PredictMethodAsync(int apiMethodId, CancellationToken cancellationToken = default)
{
var method = await _inboundApi.GetApiMethodByIdAsync(apiMethodId, cancellationToken);
if (method == null)
return PredictedFlow.Empty;
// The predictor is synchronous and takes a synchronous resolver, and this method runs on the
// Blazor circuit's single-threaded renderer sync context — so the instance → template → script
// lookups must NEVER block on the repository from inside the resolver. The original
// `.GetAwaiter().GetResult()` there deadlocked the whole circuit (every later click and JS
// interop reply queued behind the blocked dispatcher forever) the moment a method whose script
// routes to a site script was chosen; found 2026-09-06 while chasing a "login-disabled interop
// stall" that was really this. Instead the walk is repeated: each pass records the route targets
// it asked for and could not answer, those are awaited between passes, and the walk converges
// once a pass asks for nothing new. Cheap — a pass is a Roslyn syntax walk over a short script.
var memo = new Dictionary<(string Instance, string Script), string?>();
var script = method.Script ?? string.Empty;
for (var pass = 0; pass < MaxPredictionPasses; pass++)
{
var requested = new List<(string Instance, string Script)>();
string? Resolve(string instanceCode, string scriptName)
{
var key = (instanceCode, scriptName);
if (memo.TryGetValue(key, out var cached))
return cached;
if (!requested.Contains(key))
requested.Add(key);
return null;
}
var flow = FlowShapePredictor.PredictInboundMethod(method.Name, script, Resolve);
if (requested.Count == 0)
return flow;
foreach (var key in requested)
{
string? body;
try
{
body = await ResolveSiteScriptAsync(key.Instance, key.Script, cancellationToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
_logger.LogDebug(ex, "Flow prediction could not resolve {Instance}.{Script}", key.Instance, key.Script);
body = null;
}
memo[key] = body;
}
}
// Pathological: a walk still discovering new targets past the pass cap — predict with what is known.
return FlowShapePredictor.PredictInboundMethod(method.Name, script, (i, s) => memo.GetValueOrDefault((i, s)));
}
///
public async Task> GetInstanceScriptsAsync(int instanceId, CancellationToken cancellationToken = default)
=> (await FlattenScriptsAsync(instanceId, cancellationToken))?.Options ?? [];
///
public async Task PredictScriptAsync(int instanceId, string scriptName, CancellationToken cancellationToken = default)
{
var loaded = await CachedScriptsAsync(instanceId, cancellationToken);
var script = loaded?.Scripts.FirstOrDefault(s => s.CanonicalName.Equals(scriptName, StringComparison.OrdinalIgnoreCase));
if (loaded == null || script == null)
return PredictedFlow.Empty;
var title = $"{loaded.UniqueName}.{script.CanonicalName}";
if (ScriptTriggerAttribute.TryRead(script.TriggerType, script.TriggerConfiguration, out var attribute) && attribute != null)
return PredictValueChange(loaded, attribute);
if (ScriptTriggerAttribute.IsWriteKind(script.TriggerType))
{
// A write-kind trigger whose configuration names no attribute: nothing to write, so
// the run cannot be predicted past the script itself.
return new PredictedFlow(
[new PredictedNode("n1", FlowStepKind.ScriptRun, FlowLane.Site, title, false, "trigger names no attribute")],
[]);
}
// Call kinds: the debugger invokes the script directly through the traced routed-call
// path, so the root is a Central-lane RouteCall and the site script hangs off it across
// the cluster boundary — the same shape a `Route.To(instance).Call(script)` renders.
var nodes = new List
{
new("n1", FlowStepKind.RouteCall, FlowLane.Central, title, false, null),
};
var edges = new List();
AppendSubFlow(FlowShapePredictor.PredictSiteScript(title, script.Code ?? string.Empty), nodes, edges, crossesBoundary: true);
return new PredictedFlow(nodes, edges);
}
///
public async Task RunMethodAsync(int apiMethodId, JsonElement? parameters, CancellationToken cancellationToken = default)
{
var method = await _inboundApi.GetApiMethodByIdAsync(apiMethodId, cancellationToken);
var timeout = RunAskTimeoutFor(method?.TimeoutSeconds ?? 30);
return await SendAndBuildAsync(new FlowDebugRunMethodCommand(apiMethodId, parameters), timeout, cancellationToken);
}
///
public async Task RunScriptAsync(
int instanceId, string scriptName, JsonElement? parameters, string? value, CancellationToken cancellationToken = default)
{
// The server re-resolves the script and never trusts a client-supplied trigger kind
// (design 2026-09-08 §2); the kind is read here only to size the Ask wait — a call kind
// is bounded by the script's own execution timeout (falling back to the runner's own
// FlowDebugRunner.DefaultScriptTimeoutSeconds so client and server agree on the bound),
// a write kind by the trigger-wait window. When the script cannot be resolved at all the
// server may still take EITHER branch, so the fallback is the longer of the two — a
// client Ask shorter than the server's bound would report a phantom timeout on a run
// that is still going.
var command = new FlowDebugRunScriptCommand(instanceId, scriptName, parameters, value);
var option = (await CachedScriptsAsync(instanceId, cancellationToken))?.Options
.FirstOrDefault(o => o.CanonicalName.Equals(scriptName, StringComparison.OrdinalIgnoreCase));
var writeKindWait = ValueChangeAskTimeoutFor(command.WaitSeconds);
var defaultCallWait = RunAskTimeoutFor(FlowDebugRunner.DefaultScriptTimeoutSeconds);
var timeout = option switch
{
{ IsWriteKind: false } => RunAskTimeoutFor(option.ExecutionTimeoutSeconds ?? FlowDebugRunner.DefaultScriptTimeoutSeconds),
not null => writeKindWait,
null => writeKindWait > defaultCallWait ? writeKindWait : defaultCallWait,
};
return await SendAndBuildAsync(command, timeout, cancellationToken);
}
///
public Task ReplayAsync(Guid executionId, CancellationToken cancellationToken = default)
=> SendAndBuildAsync(new FlowDebugReplayCommand(executionId), ReplayAskTimeout, cancellationToken);
///
public Task RetryOutboundAsync(FlowStep step, CancellationToken cancellationToken = default)
=> RelayAsync(step, retry: true, cancellationToken);
///
public Task DiscardOutboundAsync(FlowStep step, CancellationToken cancellationToken = default)
=> RelayAsync(step, retry: false, cancellationToken);
///
/// The Retry / Discard relay behind and
/// : guards first (nothing leaves the node for a step
/// that cannot be acted on), then the outbox or the site-call relay by step kind, with the
/// outcome mapped to the report pages' wording. A transport fault becomes a failed result,
/// never an exception on the page.
///
private async Task RelayAsync(FlowStep step, bool retry, CancellationToken ct)
{
ArgumentNullException.ThrowIfNull(step);
var verb = retry ? "Retry" : "Discard";
if (!FlowStepProjector.IsOutbound(step.Kind))
return new(false, "Only an API call, database call or notification node can be retried or discarded.");
if (step.Outcome != FlowStepOutcome.Parked)
return new(false, "This node is not parked — nothing to retry or discard.");
if (step.CorrelationId is not Guid c)
return new(false, "This call has no tracked operation id (a sync call is not store-and-forwarded), so there is nothing to retry or discard.");
var isNotify = step.Kind == FlowStepKind.Notify;
if (!isNotify && string.IsNullOrWhiteSpace(step.SiteId))
return new(false, "The owning site of this call is unknown.");
var shortId = c.ToString("N")[..12];
try
{
var requestedBy = await _auth.RequireRoleAsync(Roles.Deployer);
var corr = Guid.NewGuid().ToString("N");
if (isNotify)
{
// Notify.Send mints NotificationId as Guid "N" (no dashes; ScriptRuntimeContext) and the
// outbox parses it back into the audit row's CorrelationId — so the relay key is "N".
var notificationId = c.ToString("N");
if (retry)
{
var r = await _comms.RetryNotificationAsync(new RetryNotificationRequest(corr, notificationId, requestedBy), ct);
return new(r.Success, r.Success
? $"Notification {shortId} re-queued for delivery."
: r.ErrorMessage ?? "Retry failed.");
}
var d = await _comms.DiscardNotificationAsync(new DiscardNotificationRequest(corr, notificationId, requestedBy), ct);
return new(d.Success, d.Success
? $"Notification {shortId} discarded."
: d.ErrorMessage ?? "Discard failed.");
}
var site = step.SiteId!;
var scope = new SiteScopeService(_auth);
if (!await scope.IsSystemWideAsync())
{
var owner = (await _sites.GetAllSitesAsync(ct)).FirstOrDefault(s => s.SiteIdentifier == site);
if (owner is null || !await scope.IsSiteAllowedAsync(owner.Id))
return new(false, "You are not permitted to act on cached calls for this site.");
}
SiteCallRelayOutcome outcome;
bool siteReachable;
string? errorMessage;
if (retry)
{
var r = await _comms.RetrySiteCallAsync(new RetrySiteCallRequest(corr, c, site, requestedBy), ct);
(outcome, siteReachable, errorMessage) = (r.Outcome, r.SiteReachable, r.ErrorMessage);
}
else
{
var d = await _comms.DiscardSiteCallAsync(new DiscardSiteCallRequest(corr, c, site, requestedBy), ct);
(outcome, siteReachable, errorMessage) = (d.Outcome, d.SiteReachable, d.ErrorMessage);
}
return MapRelayOutcome(outcome, siteReachable, errorMessage, $"{verb} of {shortId} relayed to {site}.");
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Flow Debugger {Verb} relay failed for {Kind} {CorrelationId}", verb, step.Kind, c);
return new(false, $"{verb} failed: {ex.Message}");
}
}
///
/// Maps a site-call relay outcome exactly as the Site Calls report page's
/// ShowRelayOutcome does: succeeds;
/// is a definitive "nothing to do";
/// — or an
/// that also reports the site unreachable —
/// gets the unreachable wording; anything else the site's own message or a generic failure.
///
private static FlowOutboundActionResult MapRelayOutcome(
SiteCallRelayOutcome outcome, bool siteReachable, string? errorMessage, string appliedMessage)
{
const string Unreachable = "Site unreachable — the relay did not reach the owning site. Try again once the site is back online.";
return outcome switch
{
SiteCallRelayOutcome.Applied => new(true, appliedMessage),
SiteCallRelayOutcome.NotParked => new(false, errorMessage ?? "The site reported nothing to do — the cached call is no longer parked."),
SiteCallRelayOutcome.SiteUnreachable => new(false, errorMessage ?? Unreachable),
SiteCallRelayOutcome.OperationFailed when !siteReachable => new(false, errorMessage ?? Unreachable),
_ => new(false, errorMessage ?? "The site could not apply the action."),
};
}
///
/// Resolves instanceCode.scriptName to a script body: instance by unique name →
/// its template chain (nearest definition wins) → the named script's code.
///
private async Task ResolveSiteScriptAsync(string instanceCode, string scriptName, CancellationToken ct)
{
var instance = await _templates.GetInstanceByUniqueNameAsync(instanceCode, ct);
if (instance == null)
return null;
var scripts = await CollectEffectiveScriptsAsync(instance.TemplateId, ct);
return scripts.TryGetValue(scriptName, out var script) ? script.Code : null;
}
///
/// Walks the template parent chain from upward, collecting
/// scripts by name with the nearest (most-derived) definition winning — so an override in
/// a derived template is what gets predicted, and a base-only script is still found.
///
private async Task> CollectEffectiveScriptsAsync(int templateId, CancellationToken ct)
{
var result = new Dictionary(StringComparer.OrdinalIgnoreCase);
int? current = templateId;
var hops = 0;
var seen = new HashSet();
while (current is int id && hops++ < MaxTemplateChain && seen.Add(id))
{
var scripts = await _templates.GetScriptsByTemplateIdAsync(id, ct);
foreach (var s in scripts)
{
if (!result.ContainsKey(s.Name))
result[s.Name] = s;
}
var template = await _templates.GetTemplateByIdAsync(id, ct);
current = template?.ParentTemplateId;
}
return result;
}
///
/// The instance's scripts as the current picker selection knows them: the cached entry a
/// call just refreshed, or a fresh flatten when the
/// prediction / run is reached without one (a deep link straight into a run).
///
private async Task CachedScriptsAsync(int instanceId, CancellationToken ct)
=> _scriptCache.TryGetValue(instanceId, out var cached) ? cached : await FlattenScriptsAsync(instanceId, ct);
///
/// Flattens an instance's scripts (design 2026-09-08 §2) — canonical names, effective trigger
/// kinds and effective parameter definitions, i.e. exactly what the site's Instance Actor keys
/// on — and refreshes the cache entry, so a template edit or a redeploy is picked up on the
/// next picker selection. Returns null when the instance does not exist (Debug — an id from a
/// stale link) or the flatten failed (Warning — the same severity DeploymentService and
/// the Configure page give it, since the instance could not be deployed either); either way
/// the stale entry is dropped rather than served on. Neither is a page error: the picker
/// simply has no scripts to offer.
///
private async Task FlattenScriptsAsync(int instanceId, CancellationToken ct)
{
var instance = await _templates.GetInstanceByIdAsync(instanceId, ct);
if (instance == null)
{
_scriptCache.Remove(instanceId);
return null;
}
var flattened = await _pipeline.FlattenAndValidateAsync(instance.Id, ct, validateScripts: false);
if (flattened.IsFailure)
{
_logger.LogWarning(
"Flow Debugger could not flatten instance {InstanceId} ({UniqueName}): {Error}",
instanceId, instance.UniqueName, flattened.Error);
_scriptCache.Remove(instanceId);
return null;
}
var scripts = flattened.Value.Configuration.Scripts
.OrderBy(s => s.CanonicalName, StringComparer.OrdinalIgnoreCase)
.ToList();
var options = scripts
.Select(s => new FlowScriptOption(
s.CanonicalName,
s.TriggerType,
ScriptTriggerAttribute.TryRead(s.TriggerType, s.TriggerConfiguration, out var a) ? a : null,
s.ParameterDefinitions,
s.ExecutionTimeoutSeconds))
.ToList();
var loaded = new InstanceScripts(instance.UniqueName, scripts, options);
_scriptCache[instanceId] = loaded;
return loaded;
}
///
/// Predicts the flow a write to would trigger: the write
/// itself on the Tags lane, then every script whose write-kind trigger names that attribute
/// predicted as a site script. Conditional triggers are included alongside ValueChange —
/// both name a single monitored attribute and both really fire on the write.
///
private static PredictedFlow PredictValueChange(InstanceScripts loaded, string attributeName)
{
var triggered = loaded.Scripts
.Where(s => ScriptTriggerAttribute.TryRead(s.TriggerType, s.TriggerConfiguration, out var a)
&& string.Equals(a, attributeName, StringComparison.OrdinalIgnoreCase))
.ToList();
var nodes = new List
{
new("n1", FlowStepKind.AttributeWrite, FlowLane.Tags, $"{loaded.UniqueName}.{attributeName}", false,
triggered.Count == 0 ? "no value-change script on this attribute" : null),
};
var edges = new List();
foreach (var script in triggered)
{
var sub = FlowShapePredictor.PredictSiteScript($"{loaded.UniqueName}.{script.CanonicalName}", script.Code ?? string.Empty);
AppendSubFlow(sub, nodes, edges, crossesBoundary: false);
}
return new PredictedFlow(nodes, edges);
}
///
/// Appends a predicted sub-flow to /,
/// renumbering its node ids into the outer sequence and linking its first node to the root
/// (n1).
///
private static void AppendSubFlow(PredictedFlow sub, List nodes, List edges, bool crossesBoundary)
{
var offset = nodes.Count;
var map = new Dictionary(StringComparer.Ordinal);
foreach (var n in sub.Nodes)
{
var id = "n" + (nodes.Count + 1);
map[n.Id] = id;
nodes.Add(n with { Id = id });
}
if (sub.Nodes.Count > 0)
edges.Add(new PredictedEdge("n1", nodes[offset].Id, crossesBoundary));
foreach (var e in sub.Edges)
edges.Add(e with { From = map[e.From], To = map[e.To] });
}
private async Task SendAndBuildAsync(object command, TimeSpan timeout, CancellationToken ct)
{
var response = await SendAsync(command, timeout, ct);
switch (response)
{
case ManagementSuccess success:
{
FlowDebugRunResult? result;
try
{
result = JsonSerializer.Deserialize(success.JsonData, ResultJsonOptions);
}
catch (JsonException ex)
{
_logger.LogError(ex, "Flow Debugger result could not be parsed for {Command}", command.GetType().Name);
return FlowDebugOutcome.Fail("The server returned an unreadable result.", "BAD_RESULT");
}
if (result == null)
return FlowDebugOutcome.Fail("The server returned no result.", "BAD_RESULT");
var graph = FlowGraphBuilder.Build(result.Events ?? [], result.Truncated, result.LogsTruncated);
return new FlowDebugOutcome(graph, result, null, null, result.LogsTruncated);
}
case ManagementUnauthorized unauthorized:
return FlowDebugOutcome.Fail(unauthorized.Message, "UNAUTHORIZED");
case ManagementError error:
return FlowDebugOutcome.Fail(error.Error, error.ErrorCode);
default:
return FlowDebugOutcome.Fail("Unexpected response from the management service.", "INTERNAL_ERROR");
}
}
///
/// Wraps in a for the current
/// principal and Asks the ManagementActor. Transport faults become a synthetic
/// so callers handle one response shape.
///
private async Task