e2e: drive each client CLI through one long-lived batch process
The cross-language e2e matrix spawned one CLI process per operation —
~250 per client — paying a process (and, for the Java CLI, a full JVM)
cold-start every time. The Java leg alone ran ~16 minutes.
Each client CLI (dotnet, go, rust, python, java) gains a `batch`
subcommand: a single process that reads one command line from stdin,
runs it through the normal subcommand dispatch, writes the JSON result,
then a line containing exactly `__MXGW_BATCH_EOR__`. A failing command
writes its `{"error":...}` envelope and the loop continues.
run-client-e2e-tests.ps1 now launches one batch process per client and
pings every operation through its stdin/stdout, so startup is paid once
per client. The orchestration and assertions are unchanged; the parity
and auth phases now read the `{"error":...}` envelope instead of a
process exit code.
Full 5-client matrix with -VerifyWrite: ~15 min, down from ~35; the Java
leg dropped from ~16 min to ~2-3.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -468,6 +468,141 @@ public sealed class MxGatewayClientCliTests
|
||||
Assert.Contains("\"objectCount\": 99", text);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode executes a single no-gateway command and writes the EOR sentinel.</summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_SingleVersionCommand_WritesOutputAndEorSentinel()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
using var stdin = new StringReader("version --json\n");
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
["batch"],
|
||||
output,
|
||||
error,
|
||||
clientFactory: null,
|
||||
standardInput: stdin);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
string text = output.ToString();
|
||||
Assert.Contains("\"gatewayProtocolVersion\"", text);
|
||||
Assert.Contains("__MXGW_BATCH_EOR__", text);
|
||||
// Sentinel must appear after the output, not before.
|
||||
int outputIdx = text.IndexOf("gatewayProtocolVersion", StringComparison.Ordinal);
|
||||
int eorIdx = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
|
||||
Assert.True(outputIdx < eorIdx, "EOR sentinel must follow command output.");
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode processes two commands sequentially and writes two EOR sentinels.</summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_TwoVersionCommands_WritesTwoEorSentinels()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
// Two commands followed by EOF (end of string).
|
||||
using var stdin = new StringReader("version\nversion --json\n");
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
["batch"],
|
||||
output,
|
||||
error,
|
||||
clientFactory: null,
|
||||
standardInput: stdin);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
string text = output.ToString();
|
||||
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
|
||||
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
|
||||
Assert.True(firstEor >= 0, "First EOR sentinel must be present.");
|
||||
Assert.True(secondEor > firstEor, "Second EOR sentinel must follow first.");
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode on EOF (empty stdin) exits 0 immediately without writing any sentinel.</summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_EmptyStdin_ExitsZeroWithNoOutput()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
using var stdin = new StringReader(string.Empty);
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
["batch"],
|
||||
output,
|
||||
error,
|
||||
clientFactory: null,
|
||||
standardInput: stdin);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
Assert.Equal(string.Empty, output.ToString());
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that batch mode continues after a command failure and writes the error JSON
|
||||
/// to stdout (not stderr), followed by the EOR sentinel.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_CommandFailure_WritesErrorJsonToStdoutAndContinues()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
// First line: a gateway command with no API key (will fail).
|
||||
// Second line: version (will succeed).
|
||||
using var stdin = new StringReader("open-session --endpoint http://localhost:5000\nversion --json\n");
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
["batch"],
|
||||
output,
|
||||
error,
|
||||
clientFactory: _ => throw new InvalidOperationException("injected failure"),
|
||||
standardInput: stdin);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
string text = output.ToString();
|
||||
|
||||
// Error record: the error JSON must be on stdout, not stderr.
|
||||
Assert.Contains("\"error\"", text);
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
|
||||
// Both records must be present.
|
||||
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
|
||||
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
|
||||
Assert.True(firstEor >= 0, "EOR after failed command must be present.");
|
||||
Assert.True(secondEor > firstEor, "EOR after successful command must follow first EOR.");
|
||||
|
||||
// Second record must contain the version output.
|
||||
string afterFirstEor = text[(firstEor + "__MXGW_BATCH_EOR__".Length)..];
|
||||
Assert.Contains("\"gatewayProtocolVersion\"", afterFirstEor);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode treats an empty (blank) line as EOF and exits 0.</summary>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_EmptyLine_ExitsZeroAfterPreviousCommands()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
// One command, then an empty line (stop signal), then another command that must NOT run.
|
||||
using var stdin = new StringReader("version --json\n\nversion --json\n");
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
["batch"],
|
||||
output,
|
||||
error,
|
||||
clientFactory: null,
|
||||
standardInput: stdin);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
string text = output.ToString();
|
||||
// Only one EOR sentinel — the second command after the empty line must not execute.
|
||||
int firstEor = text.IndexOf("__MXGW_BATCH_EOR__", StringComparison.Ordinal);
|
||||
int secondEor = text.IndexOf("__MXGW_BATCH_EOR__", firstEor + 1, StringComparison.Ordinal);
|
||||
Assert.True(firstEor >= 0, "One EOR sentinel must be present.");
|
||||
Assert.Equal(-1, secondEor);
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
}
|
||||
|
||||
/// <summary>Fake CLI client for testing.</summary>
|
||||
private sealed class FakeCliClient : IMxGatewayCliClient
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user