c810bd0f7c
#120: Akka ILoggingAdapter lines used to bypass Serilog and go to stdout only. - BuildHocon sets akka.loggers = [Host.Actors.SerilogAkkaLogger] and keeps akka.loglevel = INFO, so Akka Debug chatter is still filtered at the adapter. The INFO+ lines that used to reach stdout now go through the Serilog pipeline instead, with no extra volume. - Lines reach the file sinks with the node enrichers. SourceContext is the logging class's full name, AkkaSource is the actor path, Akka's own timestamp is kept, and format arguments become structured properties. - The Akka logger actor writes on its own thread, after the logging actor's LogContext is gone. To keep the ExecutionId, the two DCL actors log through ExecutionAwareLogging.GetLogger(Context), which records ExecutionLogContext.Current (a new AsyncLocal mirror of the pushed id) in an ExecutionTaggedLogMessage. The logger actor pushes that id again while it writes the line. #56 remainder (coordinator decision: no replicated schema change): - The host pipeline gains ExecutionLogCaptureSink. - While a run is traced, its step logger registers its ExecutionId: the site ScriptLogger from the launcher (plus each inline CallShared child id), and the central InboundScriptLogger from the executor. - Each tagged line becomes a ScriptLog row with LogSource "node" and the new ScriptTraceExtra.LogCategory. - The rows count against the existing MaxLogLinesPerRun cap, are clipped at 2 000 chars, ride the low-priority collector lane and pass the same redactor. Untraced runs cost one volatile read. - Lines are never copied twice. The step loggers' own forwards run inside ExecutionLogCapture.Suppress(), and the sink suppresses itself while it calls a target. The registration is declared inside the launcher's and executor's try, so it ends before the catch blocks log the outcome, and FlushDropped closes it. - FlowGraphBuilder lists node lines in FlowGraph.NodeLogs and never folds them onto a step. - The Flow Debugger shows them in a Node log card (FlowNodeLogPanel): grouped per execution under its run step, with an All | Warn + filter. Live and Replay both read the same rows, so site scope and role rules apply unchanged. - CLI --show-logs prints them as a trailing block. Tests: Commons ExecutionLogCaptureTests, ExecutionLogContext.Current, FlowGraphBuilder node lines. SiteRuntime NodeLogCaptureTests covers cross-actor capture via the launcher, untraced runs capturing nothing, no double capture of script and runtime lines or of the launcher failure line, the shared cap, collector redaction, shared-script child ids and FlushDropped closing capture. InboundNodeLogCaptureTests covers the central path. DCL ExecutionAwareLoggingTests covers the DataConnectionActor write-path Debug line being tagged and an uncorrelated line not being tagged. Host SerilogAkkaLoggerTests covers mapping, the ExecutionId crossing the logger hop, HOCON, ActorSystem boot and the Akka line reaching a traced run's step log end to end. Host LoggerConfigurationTests checks that the sink is wired. bUnit FlowNodeLogPanelTests and a CLI projection test cover the UI and CLI. Every new assertion was mutation-proven. Docs: step-logs design section 12, Component-SiteRuntime, CentralUI, InboundAPI, Host, CLI README, CLAUDE.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
283 lines
13 KiB
C#
283 lines
13 KiB
C#
using System.Globalization;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using System.Text.Json.Serialization;
|
|
using ZB.MOM.WW.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Debugging;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
|
|
|
/// <summary>
|
|
/// Compact table projection for the <c>flow-debug</c> commands.
|
|
///
|
|
/// <para>
|
|
/// The handler returns a <c>FlowDebugRunResult</c> — <c>{ rootExecutionId, completed,
|
|
/// error, errorCode, events: [ … ], truncated, … }</c> whose <c>events</c> are canonical
|
|
/// audit events (each carrying its domain fields inside a <c>detailsJson</c> string). Handed
|
|
/// to the generic renderer unprojected, the object takes the Property/Value branch and the
|
|
/// whole event array lands in one cell as raw JSON. This projects the events through the same
|
|
/// <see cref="FlowGraphBuilder"/> the Flow Debugger page uses and prints one row per STEP —
|
|
/// <c>occurred | kind | lane | target | durationMs | status | lifecycle | correlation | error | logs</c> —
|
|
/// in the order the page draws them, so a terminal user can read the swimlane top-to-bottom. An
|
|
/// outbound operation's rows (a cached call's Submit → Forwarded → Attempted → Resolve, a
|
|
/// notification's Send → Deliver) fold into ONE row whose <c>lifecycle</c> column counts them
|
|
/// and whose <c>status</c> is the latest row's (outbound-calls plan 2026-09-06, D3). <c>logs</c>
|
|
/// counts the step's attached <c>ScriptLog</c> lines (step-logs design 2026-09-09); passing
|
|
/// <c>showLogs</c> additionally prints those lines, and a failed step's exception detail, as
|
|
/// raw indented lines immediately under that step's row — <see cref="CommandHelpers"/>'s table
|
|
/// renderer special-cases a single-property <c>{ "note": … }</c> row to print it that way
|
|
/// instead of padding it into a table cell. A trailing <c>note</c> row (the same special case)
|
|
/// reports a failed / incomplete run (with the error code verbatim, e.g. <c>STANDBY_NODE</c>)
|
|
/// or a truncated event list — and, separately, an inline log-line cap (<c>logsTruncated</c>).
|
|
/// </para>
|
|
///
|
|
/// <para>JSON output is never touched — <c>--format json</c> gets the full raw result.</para>
|
|
/// </summary>
|
|
internal static class FlowDebugTableProjection
|
|
{
|
|
/// <summary>
|
|
/// Projects a <c>FlowDebugRunResult</c> body to the row array the table renderer
|
|
/// consumes. Returns the input unchanged when it is not JSON or not that shape, so the
|
|
/// renderer's own fallbacks still apply.
|
|
/// </summary>
|
|
/// <param name="json">The raw success JSON body from the management API.</param>
|
|
/// <param name="showLogs">
|
|
/// When true, prints each step's <c>ScriptLog</c> lines and (on a failed step) the first
|
|
/// three lines of its exception detail as additional indented <c>note</c> rows immediately
|
|
/// following that step's row. The <c>logs</c>
|
|
/// column (a count of attached log lines) is always present, independent of this flag.
|
|
/// </param>
|
|
/// <returns>Compact JSON suitable for table rendering.</returns>
|
|
internal static string ProjectEvents(string json, bool showLogs = false)
|
|
{
|
|
JsonDocument doc;
|
|
try
|
|
{
|
|
doc = JsonDocument.Parse(json);
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return json;
|
|
}
|
|
|
|
using (doc)
|
|
{
|
|
var root = doc.RootElement;
|
|
if (root.ValueKind != JsonValueKind.Object
|
|
|| !TryGetPropertyCI(root, "events", out var events)
|
|
|| events.ValueKind != JsonValueKind.Array)
|
|
{
|
|
return json;
|
|
}
|
|
|
|
var truncated = TryGetPropertyCI(root, "truncated", out var t) && t.ValueKind == JsonValueKind.True;
|
|
// Step logs (design 2026-09-09 D7): the inline copy sheds low-priority ScriptLog rows
|
|
// BEFORE it sheds step rows, so `logsTruncated` is a separate signal from `truncated`
|
|
// and needs its own note — the audit rows are complete, so a replay shows every line.
|
|
var logsTruncated = TryGetPropertyCI(root, "logsTruncated", out var lt)
|
|
&& lt.ValueKind == JsonValueKind.True;
|
|
|
|
var rows = ProjectSteps(events, truncated, showLogs);
|
|
if (rows == null)
|
|
{
|
|
// The event array did not deserialize as canonical audit events (a future wire
|
|
// shape, or a proxy body) — fall back to one row per raw event so nothing is hidden.
|
|
rows = new JsonArray();
|
|
foreach (var evt in events.EnumerateArray())
|
|
rows.Add(ProjectEvent(evt));
|
|
}
|
|
|
|
var completed = TryGetPropertyCI(root, "completed", out var c) && c.ValueKind == JsonValueKind.True;
|
|
var errorCode = StrValue(root, "errorCode");
|
|
var error = StrValue(root, "error");
|
|
|
|
var notes = new List<string>();
|
|
if (!completed || errorCode != null || error != null)
|
|
{
|
|
notes.Add(errorCode != null
|
|
? $"{errorCode}: {error ?? "run did not complete"}"
|
|
: error ?? "run did not complete");
|
|
}
|
|
|
|
if (truncated)
|
|
notes.Add("event list truncated — the audit rows remain complete (audit tree --execution-id <root>)");
|
|
|
|
// The page shows this; the CLI silently dropped it.
|
|
if (logsTruncated)
|
|
notes.Add("inline log lines capped — replay pulls the rest");
|
|
|
|
if (notes.Count > 0)
|
|
rows.Add(new JsonObject { ["note"] = JsonValue.Create(string.Join("; ", notes)) });
|
|
|
|
return rows.ToJsonString();
|
|
}
|
|
}
|
|
|
|
private static readonly JsonSerializerOptions EventOptions = new(JsonSerializerDefaults.Web)
|
|
{
|
|
Converters = { new JsonStringEnumConverter() },
|
|
};
|
|
|
|
/// <summary>
|
|
/// Builds the flow graph from the canonical events and projects one row per step. Returns
|
|
/// null when the array is not a list of canonical audit events.
|
|
/// </summary>
|
|
private static JsonArray? ProjectSteps(JsonElement events, bool truncated, bool showLogs)
|
|
{
|
|
List<AuditEvent>? list;
|
|
try
|
|
{
|
|
list = JsonSerializer.Deserialize<List<AuditEvent>>(events.GetRawText(), EventOptions);
|
|
}
|
|
catch (JsonException)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if (list == null || list.Any(e => e == null || e.EventId == Guid.Empty))
|
|
return null;
|
|
|
|
// Status per event: the graph's outcome is a coarser projection; the table keeps the
|
|
// audit status word (Delivered / Submitted / Parked …) the operator sees elsewhere.
|
|
var statusById = new Dictionary<Guid, string?>();
|
|
foreach (var e in list)
|
|
statusById[e.EventId] = AuditDetailsCodec.Deserialize(e.DetailsJson).Status;
|
|
|
|
var graph = FlowGraphBuilder.Build(list, truncated);
|
|
var rows = new JsonArray();
|
|
foreach (var step in graph.Steps)
|
|
{
|
|
var lifecycle = step.Lifecycle;
|
|
var status = lifecycle is { Count: > 0 }
|
|
? lifecycle[^1].Status
|
|
: statusById.TryGetValue(step.EventId, out var st) ? st : step.Outcome.ToString();
|
|
|
|
var target = step.Title;
|
|
if (step.Lane != FlowLane.External && step.Instance != null && !target.StartsWith(step.Instance, StringComparison.Ordinal))
|
|
target = $"{step.Instance}.{target}";
|
|
|
|
rows.Add(new JsonObject
|
|
{
|
|
["occurred"] = step.OccurredAtUtc.ToUniversalTime().ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture),
|
|
["kind"] = step.Kind.ToString(),
|
|
["lane"] = step.Lane.ToString(),
|
|
["target"] = target,
|
|
["durationMs"] = step.DurationMs,
|
|
["status"] = status,
|
|
["lifecycle"] = lifecycle is { Count: > 0 } ? $"{lifecycle.Count} {(lifecycle.Count == 1 ? "row" : "rows")}" : null,
|
|
["correlation"] = step.CorrelationId?.ToString("D"),
|
|
["error"] = step.ErrorMessage,
|
|
["logs"] = step.Logs?.Count ?? 0,
|
|
});
|
|
|
|
if (!showLogs)
|
|
continue;
|
|
|
|
if (step.Logs is { Count: > 0 } logs)
|
|
{
|
|
foreach (var line in logs)
|
|
rows.Add(new JsonObject { ["note"] = JsonValue.Create(FormatLogLine(line)) });
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(step.ErrorDetail))
|
|
{
|
|
foreach (var line in FormatErrorDetailLines(step.ErrorDetail))
|
|
rows.Add(new JsonObject { ["note"] = JsonValue.Create(line) });
|
|
}
|
|
}
|
|
|
|
// ScadaBridge#56: other actors' lines for the traced executions are not on any step; with
|
|
// --show-logs they print as one trailing block, each line naming its logger.
|
|
if (showLogs && graph.NodeLogs is { Count: > 0 } nodeLogs)
|
|
{
|
|
rows.Add(new JsonObject { ["note"] = JsonValue.Create($"node log ({nodeLogs.Count} line{(nodeLogs.Count == 1 ? "" : "s")}):") });
|
|
foreach (var line in nodeLogs)
|
|
rows.Add(new JsonObject { ["note"] = JsonValue.Create(FormatNodeLogLine(line)) });
|
|
}
|
|
|
|
return rows;
|
|
}
|
|
|
|
/// <summary>Formats one node-log line as <c> HH:mm:ss.fff LEVEL [Category] message</c>, the category without its namespace.</summary>
|
|
private static string FormatNodeLogLine(FlowLogLine line)
|
|
{
|
|
var time = line.OccurredAtUtc.ToUniversalTime().ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture);
|
|
var level = (line.Level ?? "info").ToUpperInvariant().PadRight(5);
|
|
var category = line.Category is { Length: > 0 } c ? c[(c.LastIndexOf('.') + 1)..] : "node";
|
|
return $" {time} {level} [{category}] {line.Message}";
|
|
}
|
|
|
|
/// <summary>Formats one <see cref="FlowLogLine"/> as <c> HH:mm:ss.fff LEVEL source message</c> (UTC, level upper-cased and padded to 5 chars).</summary>
|
|
private static string FormatLogLine(FlowLogLine line)
|
|
{
|
|
var time = line.OccurredAtUtc.ToUniversalTime().ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture);
|
|
var level = (line.Level ?? "info").ToUpperInvariant().PadRight(5);
|
|
return $" {time} {level} {line.Source} {line.Message}";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Formats the first three lines of an exception's <c>ToString()</c> detail — the first
|
|
/// prefixed <c> exception: </c>, the rest indented four spaces.
|
|
/// </summary>
|
|
private static IEnumerable<string> FormatErrorDetailLines(string errorDetail)
|
|
{
|
|
var lines = errorDetail.Replace("\r\n", "\n").Split('\n');
|
|
for (var i = 0; i < Math.Min(3, lines.Length); i++)
|
|
yield return i == 0 ? $" exception: {lines[i]}" : $" {lines[i]}";
|
|
}
|
|
|
|
/// <summary>Projects one canonical audit event (camelCase JSON) to its compact row (fallback path).</summary>
|
|
private static JsonNode ProjectEvent(JsonElement evt)
|
|
{
|
|
if (evt.ValueKind != JsonValueKind.Object)
|
|
return JsonValue.Create(evt.ToString())!;
|
|
|
|
var details = AuditDetailsCodec.Deserialize(StrValue(evt, "detailsJson"));
|
|
var kind = FlowStepProjector.ResolveKind(details.Channel, details.Kind);
|
|
var lane = FlowStepProjector.ResolveLane(kind, details.SourceSiteId);
|
|
|
|
string? occurred = null;
|
|
if (TryGetPropertyCI(evt, "occurredAtUtc", out var o) && o.ValueKind == JsonValueKind.String
|
|
&& o.TryGetDateTimeOffset(out var when))
|
|
{
|
|
occurred = when.ToUniversalTime().ToString("HH:mm:ss.fff", CultureInfo.InvariantCulture);
|
|
}
|
|
|
|
var target = StrValue(evt, "target");
|
|
var instance = details.SourceInstanceId;
|
|
var shownTarget = target ?? details.SourceScript;
|
|
if (instance != null && shownTarget != null && !shownTarget.StartsWith(instance, StringComparison.Ordinal))
|
|
shownTarget = $"{instance}.{shownTarget}";
|
|
|
|
return new JsonObject
|
|
{
|
|
["occurred"] = occurred,
|
|
["kind"] = kind.ToString(),
|
|
["lane"] = lane.ToString(),
|
|
["target"] = shownTarget ?? instance,
|
|
["durationMs"] = details.DurationMs,
|
|
["status"] = details.Status,
|
|
["error"] = details.ErrorMessage,
|
|
};
|
|
}
|
|
|
|
private static bool TryGetPropertyCI(JsonElement obj, string name, out JsonElement value)
|
|
{
|
|
foreach (var prop in obj.EnumerateObject())
|
|
{
|
|
if (string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
value = prop.Value;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
value = default;
|
|
return false;
|
|
}
|
|
|
|
private static string? StrValue(JsonElement obj, string name)
|
|
=> TryGetPropertyCI(obj, name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() : null;
|
|
}
|