dohertj2/wwtools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mxaccesscli: add read-batch / write-batch / subscribe-batch (JSONL input)

Browse Source 
Three new subcommands that take a JSONL file (or '-' for stdin) and reuse a
single MxSession across all entries. The big win is in write-batch: a
two-phase pipeline (Advise all -> drain DataChange to resolve types; Write
all -> drain WriteComplete) reduces wall time from N x (resolve + write_ack)
to ~max(resolve) + ~max(write_ack). Measured 38.2s -> 10.3s (~3.7x) for
four writes against the ZB dev galaxy; the saving grows with N.

Per-item continue-on-error: parse errors are collected line-by-line and
abort with exit 2 before any LMX session opens; runtime failures (resolve
timeout, bad references, coerce errors, write timeouts) get their own
results[] row with a typed `error` string and exit 1. Auth flags mirror
`mxa write` and are resolved once before Phase A.

Shared infra:
  - Mx/JsonlInputReader.cs: lazy line reader (skips blank / '#' lines),
    bare-string or {"tag":"..."} for read/sub, {"tag","value","type"?} for
    write, with array-suffix consistency check at parse time.
  - Mx/ValueCoercion.cs: new CoerceJToken(...) wrapper preserves the
    single source of truth for type vocabulary.

Docs:
  - README run examples extended for each new command.
  - docs/usage.md: new "Batch input format" subsection (shared contract),
    one section per command with envelope examples and a full
    failure-mode table for write-batch, plus a "Batch commands -
    verified live" section capturing the 2026-05-10 ZB-galaxy run and
    pipelining-timing numbers.
  - test-fixtures/ holds the exact JSONL files used in the verified-live
    run so the doc numbers are reproducible.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-05-11 08:34:46 -04:00
parent 6cde4d9fe4
commit 2e937228a0
10 changed files with 1260 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mxaccesscli/src/MxAccess.Cli/Mx/JsonlInputReader.cs
+209
View File
@@ -0,0 +1,209 @@
using System;
using System.Collections.Generic;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace MxAccess.Cli.Mx
{
/// JSONL parser shared by the batch commands. Lines are read lazily so
/// huge inputs don't allocate up front; per-line parsing is delegated to
/// `ParseReadEntry` / `ParseWriteEntry` so the read loop can defer parse
/// errors to a context where it can attribute them to a line number.
///
/// Line conventions (same across read / write / subscribe):
/// - Blank lines and lines starting with `#` are skipped (line numbers
/// still count them so error messages stay accurate against the file).
/// - Path `-` reads from stdin until EOF.
public static class JsonlInputReader
{
public static IEnumerable<(int LineNumber, string Json)> ReadLines(string pathOrDash)
{
if (string.IsNullOrEmpty(pathOrDash))
throw new InvalidBatchInputException("Input path must be non-empty (use '-' for stdin).");
TextReader reader = pathOrDash == "-"
? Console.In
: OpenFile(pathOrDash);
try
{
int lineNo = 0;
string raw;
while ((raw = reader.ReadLine()) != null)
{
lineNo++;
var trimmed = raw.Trim();
if (trimmed.Length == 0) continue;
if (trimmed[0] == '#') continue;
yield return (lineNo, trimmed);
}
}
finally
{
// Don't close Console.In — the host CLR owns it and closing it
// breaks anything that reads stdin after this command returns.
if (pathOrDash != "-") reader.Dispose();
}
}
private static TextReader OpenFile(string path)
{
try
{
return new StreamReader(path);
}
catch (FileNotFoundException)
{
throw new InvalidBatchInputException($"Input file not found: {path}");
}
catch (DirectoryNotFoundException)
{
throw new InvalidBatchInputException($"Input file not found: {path}");
}
catch (IOException ex)
{
throw new InvalidBatchInputException($"Could not open input file '{path}': {ex.Message}");
}
catch (UnauthorizedAccessException ex)
{
throw new InvalidBatchInputException($"Could not open input file '{path}': {ex.Message}");
}
}
/// Parse a read / subscribe entry. Accepts either a bare JSON string
/// (the tag reference) or a `{"tag":"..."}` object. Throws
/// `InvalidBatchEntryException` on malformed input — the caller is
/// expected to surface line number + reason in the result envelope.
public static BatchReadEntry ParseReadEntry(int lineNumber, string json)
{
JToken token;
try
{
token = JToken.Parse(json);
}
catch (JsonReaderException ex)
{
throw new InvalidBatchEntryException(lineNumber, $"invalid JSON: {ex.Message}");
}
string tag;
switch (token.Type)
{
case JTokenType.String:
tag = (string)token;
break;
case JTokenType.Object:
var tagTok = token["tag"];
if (tagTok == null || tagTok.Type != JTokenType.String)
throw new InvalidBatchEntryException(lineNumber,
"object entry must have a 'tag' string field");
tag = (string)tagTok;
break;
default:
throw new InvalidBatchEntryException(lineNumber,
"expected a tag string or an object with a 'tag' field");
}
if (string.IsNullOrWhiteSpace(tag))
throw new InvalidBatchEntryException(lineNumber, "tag must be non-empty");
return new BatchReadEntry { Tag = tag, LineNumber = lineNumber };
}
/// Parse a write entry. Required: `tag` (string), `value` (any JSON
/// type). Optional: `type` (string, mirrors `mxa write --type`).
/// If `value` is a JSON array the tag must end in `[]` (matches the
/// array-write rule in `WriteCommand`).
public static BatchWriteEntry ParseWriteEntry(int lineNumber, string json)
{
JToken token;
try
{
token = JToken.Parse(json);
}
catch (JsonReaderException ex)
{
throw new InvalidBatchEntryException(lineNumber, $"invalid JSON: {ex.Message}");
}
if (token.Type != JTokenType.Object)
throw new InvalidBatchEntryException(lineNumber,
"write entry must be a JSON object with at least 'tag' and 'value'");
var tagTok = token["tag"];
var valueTok = token["value"];
var typeTok = token["type"];
if (tagTok == null || tagTok.Type != JTokenType.String)
throw new InvalidBatchEntryException(lineNumber, "'tag' must be a non-empty string");
var tag = (string)tagTok;
if (string.IsNullOrWhiteSpace(tag))
throw new InvalidBatchEntryException(lineNumber, "'tag' must be a non-empty string");
if (valueTok == null)
throw new InvalidBatchEntryException(lineNumber, "'value' is required");
string typeHint = null;
if (typeTok != null)
{
if (typeTok.Type != JTokenType.String)
throw new InvalidBatchEntryException(lineNumber, "'type' must be a string when present");
typeHint = (string)typeTok;
}
bool tagIsArray = tag.EndsWith("[]", StringComparison.Ordinal);
bool valueIsArray = valueTok.Type == JTokenType.Array;
if (valueIsArray && !tagIsArray)
throw new InvalidBatchEntryException(lineNumber,
"array 'value' requires a tag ending in '[]' (whole-array write reference)");
if (!valueIsArray && tagIsArray)
throw new InvalidBatchEntryException(lineNumber,
"tag ends in '[]' (array write) but 'value' is not a JSON array");
return new BatchWriteEntry
{
Tag = tag,
RawValue = valueTok,
TypeHint = typeHint,
IsArray = tagIsArray,
LineNumber = lineNumber,
};
}
}
public sealed class BatchReadEntry
{
public string Tag { get; init; }
public int LineNumber { get; init; }
}
public sealed class BatchWriteEntry
{
public string Tag { get; init; }
public JToken RawValue { get; init; }
public string TypeHint { get; init; }
public bool IsArray { get; init; }
public int LineNumber { get; init; }
}
/// Thrown for problems with the file itself (missing, unreadable, etc).
/// Distinct from `InvalidBatchEntryException` so the command can choose
/// to abort the whole batch on this category without trying to parse.
public sealed class InvalidBatchInputException : Exception
{
public InvalidBatchInputException(string message) : base(message) { }
}
/// Thrown for problems with a single line. Always carries the 1-based line
/// number so the command can echo it back in the envelope.
public sealed class InvalidBatchEntryException : Exception
{
public int LineNumber { get; }
public InvalidBatchEntryException(int lineNumber, string reason) : base(reason)
{
LineNumber = lineNumber;
}
}
}
mxaccesscli/src/MxAccess.Cli/Mx/ValueCoercion.cs
+36
View File
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using Newtonsoft.Json.Linq;
namespace MxAccess.Cli.Mx
{
@@ -90,6 +91,41 @@ namespace MxAccess.Cli.Mx
}
}
/// JSON-token entry point used by `read-batch` / `write-batch`. Renders
/// each token to the string form the existing `Coerce` / `CoerceArray`
/// dispatch table accepts, then forwards — so the type vocabulary
/// stays defined in one place.
public static object CoerceJToken(JToken raw, string typeHint, bool isArray)
{
if (raw == null) throw new ArgumentNullException(nameof(raw));
if (isArray)
{
if (raw.Type != JTokenType.Array)
throw new ArgumentException("isArray=true but JSON token is not an array.");
var rendered = new List<string>();
foreach (var el in (JArray)raw) rendered.Add(RenderToken(el));
return CoerceArray(rendered, typeHint);
}
return Coerce(RenderToken(raw), typeHint);
}
private static string RenderToken(JToken t)
{
switch (t.Type)
{
case JTokenType.String: return (string)t;
case JTokenType.Boolean: return ((bool)t) ? "true" : "false";
case JTokenType.Integer: return ((long)t).ToString(CultureInfo.InvariantCulture);
case JTokenType.Float: return ((double)t).ToString("R", CultureInfo.InvariantCulture);
case JTokenType.Date: return ((DateTime)t).ToString("o", CultureInfo.InvariantCulture);
case JTokenType.Null:
throw new ArgumentException("JSON null is not a writable value.");
default:
throw new ArgumentException($"Cannot coerce JSON token of type {t.Type} to a write value.");
}
}
private static T[] Convert<T>(IReadOnlyList<string> raw, Func<string, T> parse)
{
var arr = new T[raw.Count];