e2e: drive each client CLI through one long-lived batch process

The cross-language e2e matrix spawned one CLI process per operation —
~250 per client — paying a process (and, for the Java CLI, a full JVM)
cold-start every time. The Java leg alone ran ~16 minutes.

Each client CLI (dotnet, go, rust, python, java) gains a `batch`
subcommand: a single process that reads one command line from stdin,
runs it through the normal subcommand dispatch, writes the JSON result,
then a line containing exactly `__MXGW_BATCH_EOR__`. A failing command
writes its `{"error":...}` envelope and the loop continues.

run-client-e2e-tests.ps1 now launches one batch process per client and
pings every operation through its stdin/stdout, so startup is paid once
per client. The orchestration and assertions are unchanged; the parity
and auth phases now read the `{"error":...}` envelope instead of a
process exit code.

Full 5-client matrix with -VerifyWrite: ~15 min, down from ~35; the Java
leg dropped from ~16 min to ~2-3.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Joseph Doherty
2026-05-21 06:20:13 -04:00
parent c1ff8c94e8
commit 6126099cdb
10 changed files with 970 additions and 47 deletions
@@ -468,6 +468,141 @@ public sealed class MxGatewayClientCliTests
Assert.Contains("\"objectCount\": 99", text);
}
/// <summary>Verifies that batch mode executes a single no-gateway command and writes the EOR sentinel.</summary>
[Fact]
public async Task RunAsync_Batch_SingleVersionCommand_WritesOutputAndEorSentinel()
{
using var output = new StringWriter();
using var error = new StringWriter();
using var stdin = new StringReader("version --json\n");
int exitCode = await MxGatewayClientCli.RunAsync(
["batch"],
output,
error,
clientFactory: null,
standardInput: stdin);
Assert.Equal(0, exitCode);
string text = output.ToString();
Assert.Contains("\"gatewayProtocolVersion\"", text);
Assert.Contains("__MXGW_BATCH_EOR__", text);
// Sentinel must appear after the output, not before.
int outputIdx = text.IndexOf("gatewayProtocolVersion", StringComparison.Ordinal);
int eorIdx = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
Assert.True(outputIdx < eorIdx, "EOR sentinel must follow command output.");
Assert.Equal(string.Empty, error.ToString());
}
/// <summary>Verifies that batch mode processes two commands sequentially and writes two EOR sentinels.</summary>
[Fact]
public async Task RunAsync_Batch_TwoVersionCommands_WritesTwoEorSentinels()
{
using var output = new StringWriter();
using var error = new StringWriter();
// Two commands followed by EOF (end of string).
using var stdin = new StringReader("version\nversion --json\n");
int exitCode = await MxGatewayClientCli.RunAsync(
["batch"],
output,
error,
clientFactory: null,
standardInput: stdin);
Assert.Equal(0, exitCode);
string text = output.ToString();
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
Assert.True(firstEor >= 0, "First EOR sentinel must be present.");
Assert.True(secondEor > firstEor, "Second EOR sentinel must follow first.");
Assert.Equal(string.Empty, error.ToString());
}
/// <summary>Verifies that batch mode on EOF (empty stdin) exits 0 immediately without writing any sentinel.</summary>
[Fact]
public async Task RunAsync_Batch_EmptyStdin_ExitsZeroWithNoOutput()
{
using var output = new StringWriter();
using var error = new StringWriter();
using var stdin = new StringReader(string.Empty);
int exitCode = await MxGatewayClientCli.RunAsync(
["batch"],
output,
error,
clientFactory: null,
standardInput: stdin);
Assert.Equal(0, exitCode);
Assert.Equal(string.Empty, output.ToString());
Assert.Equal(string.Empty, error.ToString());
}
/// <summary>
/// Verifies that batch mode continues after a command failure and writes the error JSON
/// to stdout (not stderr), followed by the EOR sentinel.
/// </summary>
[Fact]
public async Task RunAsync_Batch_CommandFailure_WritesErrorJsonToStdoutAndContinues()
{
using var output = new StringWriter();
using var error = new StringWriter();
// First line: a gateway command with no API key (will fail).
// Second line: version (will succeed).
using var stdin = new StringReader("open-session --endpoint http://localhost:5000\nversion --json\n");
int exitCode = await MxGatewayClientCli.RunAsync(
["batch"],
output,
error,
clientFactory: _ => throw new InvalidOperationException("injected failure"),
standardInput: stdin);
Assert.Equal(0, exitCode);
string text = output.ToString();
// Error record: the error JSON must be on stdout, not stderr.
Assert.Contains("\"error\"", text);
Assert.Equal(string.Empty, error.ToString());
// Both records must be present.
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
Assert.True(firstEor >= 0, "EOR after failed command must be present.");
Assert.True(secondEor > firstEor, "EOR after successful command must follow first EOR.");
// Second record must contain the version output.
string afterFirstEor = text[(firstEor + "__MXGW_BATCH_EOR__".Length)..];
Assert.Contains("\"gatewayProtocolVersion\"", afterFirstEor);
}
/// <summary>Verifies that batch mode treats an empty (blank) line as EOF and exits 0.</summary>
[Fact]
public async Task RunAsync_Batch_EmptyLine_ExitsZeroAfterPreviousCommands()
{
using var output = new StringWriter();
using var error = new StringWriter();
// One command, then an empty line (stop signal), then another command that must NOT run.
using var stdin = new StringReader("version --json\n\nversion --json\n");
int exitCode = await MxGatewayClientCli.RunAsync(
["batch"],
output,
error,
clientFactory: null,
standardInput: stdin);
Assert.Equal(0, exitCode);
string text = output.ToString();
// Only one EOR sentinel — the second command after the empty line must not execute.
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
Assert.True(firstEor >= 0, "One EOR sentinel must be present.");
Assert.Equal(-1, secondEor);
Assert.Equal(string.Empty, error.ToString());
}
/// <summary>Fake CLI client for testing.</summary>
private sealed class FakeCliClient : IMxGatewayCliClient
{