dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(management): FlowDebugRunner — transport faults still write the root row; write-kind completion is script-specific; standby/unreadable-trigger tests

Browse Source 
Claude-Session: https://claude.ai/code/session_01NcMG6xXdsJbmD2wYAgkK29
This commit is contained in:
Joseph Doherty
2026-09-08 01:42:17 -04:00
parent a2bfc79e3a
commit ca9be68d91
2 changed files with 151 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.ManagementService/FlowDebug/FlowDebugRunner.cs
+53 -18
View File
@@ -53,6 +53,15 @@ public sealed class FlowDebugRunner
/// </summary>
public const int DefaultScriptTimeoutSeconds = 30;
/// <summary>
/// Ceiling on a write-kind run's spawned-run wait, whatever the command asks for. It must stay
/// UNDER <c>ManagementServiceOptions.LongRunningCommandTimeout</c> (5 minutes by default), which
/// is the Ask the management endpoint wraps this command in: a longer wait would leave the poll
/// loop running on the node after the caller's Ask has already expired, holding its scoped
/// provider and pulling from the site for a result nobody can receive.
/// </summary>
public const int MaxWaitSeconds = 300;
/// <summary>
/// Arch-review F2: <see cref="InboundScriptExecutor"/>'s code for a routed call refused
/// because the target site is outside the operator's site scope. InboundAPI cannot
@@ -304,7 +313,9 @@ public sealed class FlowDebugRunner
DateTimeOffset.UtcNow, ParentExecutionId: root, Trace: true);
RouteToCallResponse response;
var timedOut = false;
// Null while the site ANSWERED (a success or a Success=false reply); otherwise the code to
// report for a routed Ask that threw instead of answering.
string? faultCode = null;
using (var runCts = CancellationTokenSource.CreateLinkedTokenSource(ct))
{
runCts.CancelAfter(timeout + RunGrace);
@@ -314,10 +325,27 @@ public sealed class FlowDebugRunner
}
catch (OperationCanceledException) when (!ct.IsCancellationRequested)
{
timedOut = true;
faultCode = TimeoutCode;
response = new RouteToCallResponse(
request.CorrelationId, false, null, TimeoutMessage(timeout), DateTimeOffset.UtcNow);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
// RouteToCallAsync is an Akka Ask bounded by CommunicationOptions.IntegrationTimeout,
// so it throws AskTimeoutException (NOT an OperationCanceledException) when the site
// never answers — and any transport fault throws too. "No answer" is NOT "did not
// happen": the site may already have run the script. Letting the exception escape
// would lose the root row entirely and show the operator MapFault's generic internal
// error for a call that ran, so it is converted into a failed response the same way
// RouteHelper.CallAsync types its Ask faults and WriteAndPollAsync reports a failed
// write. A cancellation of the CALLER's token still propagates (filtered out above).
faultCode = ex is Akka.Actor.AskTimeoutException ? TimeoutCode : CallFailedCode;
_logger.LogWarning(ex,
"Flow Debugger routed call to site {Site} for {Instance}.{Script} faulted",
site.SiteIdentifier, instance.UniqueName, script.CanonicalName);
response = new RouteToCallResponse(
request.CorrelationId, false, null, ex.Message, DateTimeOffset.UtcNow);
}
}
// Everything from here to WriteCentralAsync runs AFTER the site already executed the
@@ -366,7 +394,7 @@ public sealed class FlowDebugRunner
ordered.AddRange(events);
return new FlowDebugRunResult(
root, response.Success, response.ErrorMessage,
response.Success ? null : timedOut ? TimeoutCode : CallFailedCode,
response.Success ? null : faultCode ?? CallFailedCode,
ordered, Truncated: collector.Truncated, startedAt, DateTimeOffset.UtcNow);
}
@@ -424,11 +452,26 @@ public sealed class FlowDebugRunner
[redactedRoot], false, startedAt, DateTimeOffset.UtcNow);
}
// The run that COUNTS as completion. For the script entry (design 2026-09-08 §4.3) that is
// the script the operator chose, matched on Target — the site stamps ScriptRunLauncher's
// `target: scriptName` on the row. Other scripts the same write happens to fire are still
// collected below (the graph draws the siblings) but must never report the CHOSEN script as
// having run. triggerScript == null is the attribute-level command, which expects no
// particular script, so any spawned run satisfies it.
bool IsAwaitedRun(AuditEvent e)
{
var d = AuditDetailsCodec.Deserialize(e.DetailsJson);
if (d.Kind != nameof(AuditKind.ScriptRun) || d.ParentExecutionId != root) return false;
return triggerScript is null
|| string.Equals(e.Target, triggerScript, StringComparison.OrdinalIgnoreCase);
}
// Poll the site for the traced run(s) the write spawned — matched by ParentExecutionId
// == root, then their children — until a ScriptRun parented to root exists AND the
// set has been quiet for SettleWindow, or WaitSeconds elapses.
// == root, then their children — until the awaited ScriptRun exists AND the set has been
// quiet for SettleWindow, or the (clamped) wait elapses.
var puller = sp.GetRequiredService<IPullTraceEventsClient>();
var deadline = _now() + TimeSpan.FromSeconds(Math.Max(1, waitSeconds));
var wait = Math.Clamp(waitSeconds, 1, MaxWaitSeconds);
var deadline = _now() + TimeSpan.FromSeconds(wait);
var known = new Dictionary<Guid, AuditEvent>();
DateTimeOffset? lastNew = null;
var since = startedAt.UtcDateTime.AddSeconds(-5);
@@ -448,11 +491,7 @@ public sealed class FlowDebugRunner
}
if (added) lastNew = _now();
var spawned = known.Values.Any(e =>
{
var d = AuditDetailsCodec.Deserialize(e.DetailsJson);
return d.Kind == nameof(AuditKind.ScriptRun) && d.ParentExecutionId == root;
});
var spawned = known.Values.Any(IsAwaitedRun);
var now = _now();
if (spawned && lastNew is { } ln && now - ln >= SettleWindow) break;
if (now >= deadline) break;
@@ -461,16 +500,12 @@ public sealed class FlowDebugRunner
var ordered = new List<AuditEvent> { redactedRoot };
ordered.AddRange(known.Values.OrderBy(e => e.OccurredAtUtc).ThenBy(e => e.EventId));
var completed = known.Values.Any(e =>
{
var d = AuditDetailsCodec.Deserialize(e.DetailsJson);
return d.Kind == nameof(AuditKind.ScriptRun) && d.ParentExecutionId == root;
});
var completed = known.Values.Any(IsAwaitedRun);
return new FlowDebugRunResult(
root, completed,
completed ? null : triggerScript is null
? $"No traced run was spawned within {waitSeconds}s (no value-change trigger on '{attributeName}', or no device echo)."
: $"No traced run of '{triggerScript}' was spawned within {waitSeconds}s after writing '{attributeName}' (trigger condition not met, or no device echo).",
? $"No traced run was spawned within {wait}s (no value-change trigger on '{attributeName}', or no device echo)."
: $"No traced run of '{triggerScript}' was spawned within {wait}s after writing '{attributeName}' (trigger condition not met, or no device echo).",
completed ? null : NoTriggerCode,
ordered, false, startedAt, DateTimeOffset.UtcNow);
}